User-agent: * Allow: / Crawl-delay: 5 User-agent: * Disallow: /cgi-bin Disallow: /images Disallow: /404.php Disallow: /403.php Disallow: /http://mangalamindia.com/404.php Disallow: /http://mangalamindia.com/403.php