# robots.txt to ALLOW all content to be crawled
 User-agent: *
 Disallow: 

# robots.txt to DISALLOW content to not be crawled
# User-agent: *
# Disallow: /

# Disallows cgi-bin and images (example)
# User-agent: *
# Disallow: /cgi-bin/
# Disallow: /images/

# Allows only major search engines and known friendly spiders
# Major Search Engines and Known Friendly Spiders (allowed)
# User-agent: Googlebot
# Disallow:
# User-agent: MSNBot
# Disallow:
# User-agent: Slurp
# Disallow:
# User-agent: Teoma
# Disallow:
# User-agent: Gigabot
# Disallow:
# User-agent: Scrubby
# Disallow:
# User-agent: Robozilla
# Disallow:
# Everyone Else (NOT allowed)
# User-agent: *
# Disallow: /
# Everyone Else (allowed)
# User-agent: *
# Disallow: