# by default, allow all directories to be spidered
User-agent: *
# dissallow some files and directories
Disallow: /mag/wagner
Disallow: /mag/notes
Disallow: /mag/zipped/
Disallow: /cgi-bin/
# disallow stress test
user-agent: stress-agent
Disallow: /