#robots.txt file for http://www.accessdata.fda.gov

#For all crawlers
User-agent: * 
Disallow: /scripts/cder/bmis/
Disallow: /scripts/cder/BMIS/
Disallow: /scripts/cber/cfapps/
Disallow: /scripts/cdrh/cfdocs/cfRL/
Disallow: /scripts/email/
Disallow: /scripts/ImportRefusals/ir_index.cfm


Hit-rate: 30 # wait 30 seconds before starting a new URL request default=30
Visiting-hours: 23:00EDT-05:00EDT #index this site between 11PM - 5AM EDT
Concurrent-hits: 2 # limit concurrent active URLS to 2 for each index server