# /robots.txt file for http://webcrawler.com/
# mail webmaster@webcrawler.com for constructive criticism

User-agent: Googlebot
Disallow: /admin
Disallow: /inc
Disallow: /survey
Disallow: /album
Disallow: /linkmarket
Disallow: /images
Disallow: /axel
Disallow: /private
Disallow: /dev
Disallow: /doc
Disallow: /forum2

User-agent: *
Disallow: /admin
Disallow: /inc
Disallow: /survey
Disallow: /album
Disallow: /linkmarket
Disallow: /images
Disallow: /axel
Disallow: /private
Disallow: /dev
Disallow: /doc
Disallow: /forum2