# robots.txt for http://www.uhrl.nsw.gov.au

# Valid crawlers and spiders are controlled

User-agent: Googlebot
Disallow: 
Disallow: /dev/
Disallow: /images/
Disallow: /admin/
Disallow: /www/admin/
Disallow: /www/cusFile/
Disallow: /www/Images/
Disallow: /www/include/
Disallow: /www/Js/
Disallow: /www/Style/
Disallow: /www/error_log

User-agent: Slurp
Disallow: 
Disallow: /dev/
Disallow: /images/
Disallow: /admin/
Disallow: /www/admin/
Disallow: /www/cusFile/
Disallow: /www/Images/
Disallow: /www/include/
Disallow: /www/Js/
Disallow: /www/Style/
Disallow: /www/error_log
Crawl-Delay:5

User-agent: MSNBot
Disallow: 
Disallow: /dev/
Disallow: /images/
Disallow: /admin/
Disallow: /www/admin/
Disallow: /www/cusFile/
Disallow: /www/Images/
Disallow: /www/include/
Disallow: /www/Js/
Disallow: /www/Style/
Disallow: /www/error_log
Crawl-Delay:5

# Allow all other Site scrapers

User-agent: *
Disallow: 
Disallow: /dev/
Disallow: /images/
Disallow: /admin/
Disallow: /www/admin/
Disallow: /www/cusFile/
Disallow: /www/Images/
Disallow: /www/include/
Disallow: /www/Js/
Disallow: /www/Style/
Disallow: /www/error_log

# Site scrapers that are completely disallowed

User-agent: Jeeves*
Disallow: /

User-agent: *libwww* 
Disallow: /

User-agent: HttpTool* 
Disallow: /

User-agent: *Fetch* 
Disallow: /

User-agent: http://www.almaden.ibm.com/cs/crawler 
Disallow: /

User-agent: BorderManager*
Disallow: /

User-agent: Maxthon
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: Bilbo/2.3b-UNIX
Disallow: /

User-agent: webcollage/1.77
Disallow: /

User-agent: FAST-WebCrawler/3.6/FirstPage
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: Java*
Disallow: /

User-agent: Offline Explorer/1.9
Disallow: /

User-agent: Missigua Locator 1.9
Disallow: /

User-agent: grub-client 
Disallow: /

User-agent: msrbot
Disallow: /

User-agent: lwp*
Disallow: /

User-agent: *newsexpress*
Disallow: /