# robots.txt # $Id: robots.txt,v 1.0 2004/10/18 00:00:00 $ # "Any empty [Disallow] value, indicates that all URLs can be retrieved. # At least one Disallow field needs to be present in a record." User-agent: Mediapartners-Google* Disallow: User-agent: Slurp Disallow: User-Agent: OmniExplorer_Bot Crawl-delay: 100 #Disallow: / User-agent: Yahoo-NewsCrawler* Disallow: User-Agent: msnbot Crawl-delay: 10 Disallow: User-agent: * Crawl-delay: 10 Disallow: /img-host