# Robots.txt file created by http://www.webtoolcentral.com
# Checked by: http://tool.motoricerca.info/robots-checker.phtml
# Alternate creator: http://www.mcanerin.com/EN/search-engine/robots-txt.asp
# For domain: http://www.sethi.org
# Reference: http://pageresource.com/zine/robotstxt.htm and http://www.searchtools.com/robots/robots-txt.html

# 
# CHECK robots.txt here: 
#   http://phpweby.com/services/robots
#   http://www.searchenginepromotionhelp.com/m/robots-text-tester/robots-checker.php
#   http://www.google.com/support/webmasters/bin/answer.py?hl=en&answer=156449
# 


# 2015-07-15: Set basic rules for ALL spiders: http://www.fusionbot.com/faqs/faq21.asp
# Previous edit was on 2013-08-30
User-agent: *
Disallow: /archives/
Disallow: /cgi-bin/
Disallow: /classes/
Disallow: /classes/class_stuff/
Disallow: /classes/class_storage_stuff/
Disallow: /contact/
Disallow: /dev/
Disallow: /iom540/
Disallow: /misc/
Disallow: /ricky/
Disallow: /ssi/
Disallow: /tmp/
Disallow: /utils/



# Apply to just the Wayback Machine: http://www.archive.org/about/exclude.php
# Check for http://www.sethi.org/investments/darvas/darvas.phps
User-agent: ia_archiver
Disallow: /cgi-bin/
Disallow: /investments/
Disallow: /tools/
Disallow: /genealogy/
Disallow: /classes/class_stuff/
Disallow: /classes/class_storage_stuff/
Disallow: /dev/
Disallow: /archives/



# Disallow wget also?
#;User-agent: wget
#;Disallow: /

# Disallow Scooter/1.0
#;User-agent: Scooter/1.0
#;Disallow: /

# Disallow Bilbo/1.2+WAP
#;User-agent: Bilbo/1.2+WAP
#;Disallow: /


# Allow these bots to get at everything:
# robots.txt generated at www.mcanerin.com
# Google
User-agent: Googlebot
Disallow: /cgi-bin/
Disallow: /classes/class_stuff/
Disallow: /classes/class_storage_stuff/
Disallow: /dev/

# MSN
User-agent: MSNBot
Disallow: /cgi-bin/
Disallow: /classes/class_stuff/
Disallow: /classes/class_storage_stuff/
Disallow: /dev/

# Yahoo
User-agent: Slurp
Disallow: /cgi-bin/
Disallow: /classes/class_stuff/
Disallow: /classes/class_storage_stuff/
Disallow: /dev/

# Ask/Teoma
User-agent: Teoma
Disallow: /cgi-bin/
Disallow: /classes/class_stuff/
Disallow: /classes/class_storage_stuff/
Disallow: /dev/

# DMOZ
User-agent: Robozilla
Disallow: /cgi-bin/
Disallow: /classes/class_stuff/
Disallow: /classes/class_storage_stuff/
Disallow: /dev/


# All robots will spider the domain
User-agent: *
# Disallow directory /analog/
Disallow: /analog/
# Disallow directory /cgi-bin/
Disallow: /cgi-bin/
# Disallow directory /guestbook/ (except for exceptions above):
Disallow: /guestbook/
# Disallow directory /utils/
Disallow: /utils/
# Disallow directory /classes/class_stuff/
Disallow: /classes/class_stuff/
Disallow: /classes/class_storage_stuff/
# Disallow directory /dev/
Disallow: /dev/

Crawl-delay: 120