# The following allows all robots to visit all files because the wildcard "*" specifies all robots. 
# User-agent: *
# Disallow:

# This one keeps all robots out. 
# User-agent: *
# Disallow: /

# The next one bars all robots from the cgi-bin and images directories: 
# User-agent: *
# Disallow: /cgi-bin/
# Disallow: /images/

# This one bans Roverdog from all files on the server: 
# User-agent: Roverdog
# Disallow: /

# This one bans keeps googlebot from getting at the cheese.htm file: 
# User-agent: googlebot
# Disallow: cheese.htm

# For more complex examples, try retrieving some of the robots.txt files from the big sites like Cnn, or Looksmart.

User-agent: *
Disallow: /_derived/
Disallow: /_fpclass/
Disallow: /_private/
Disallow: /_vti_cnf/
Disallow: /_vti_pvt/
Disallow: /audio/
Disallow: /cgi-bin/
Disallow: /data/
Disallow: /dev/
Disallow: /eXPansive/
Disallow: /eXPerience/
Disallow: /fpdb/
Disallow: /images/
Disallow: /logon/
Disallow: /old/
Disallow: /preview/
Disallow: /vhosts/
Disallow: /video/
Disallow: resume.doc
Disallow: resume.pdf