# robots.txt for http://www.anyTitle.com/

#7-8-07 added voila delay of 15, added  msnbot delay: 10, added Googlebot delay: 6
# set delay to 40: 4/5/08

User-agent: *
#Disallow: /cgi-bin/ # scripts only
Disallow: /cgi-logs/ # nothing to see
Disallow: /_logs/ # test area
Disallow: /images/ # 
Disallow: /reports/ # 
Disallow: /counter/ # 
Disallow: /icon/ # 
Disallow: /_moonies/ # 

User-Agent: sitecheck.internetseer.com
Disallow: /

User-agent: Mediapartners-Google*
Crawl-delay: 10
# is 5 seconds
#Disallow:

User-agent: ShopWiki
Crawl-delay: 40
# is 5 seconds

User-agent: msnbot
Crawl-delay: 40

User-Agent: Googlebot
Crawl-delay: 10

User-Agent: Twiceler
Crawl-delay: 40

User-agent: VoilaBot
Crawl-delay: 40
Disallow: /cgi-bin
Disallow: /test/ # test area
Disallow: /demo/ # client demos
Disallow: /testarea/ # client demos
Disallow: /tda/ # 
Disallow: /Scroll/ # 

User-agent: Gigabot
Crawl-delay: 10
# is 5 seconds

User-agent: Slurp #yahoo
Crawl-delay: 50