# /robots.txt file for http://webcrawler.com/

User-agent: *
Disallow: /admin
Disallow: /adodb
Disallow: /_archive
Disallow: /presenter_reports
Disallow: /report
Disallow: /report_2003
Disallow: /community
Disallow: /_mm/
Disallow: /_notes/
Disallow: /_baks/
Disallow: /MMWIP/

User-agent: googlebot
Disallow: *.csi