# All robots will spider the domain
User-agent: *
Disallow: 
Disallow: /_private/
Disallow: /_vti_bin/
Disallow: /_vti_cnf/
Disallow: /_vti_log/
Disallow: /_vti_pvt/
Disallow: /awstats/
Disallow: /_vti_log/
Disallow: /cgi-bin/
Disallow: /rainbow/
Disallow: /search/
Disallow: /webalizer/
Disallow: /tempdata/
Disallow: /tempdata1/
Disallow: /tempdata2/