#robots.txt file for http://www.accessdata.fda.gov

User-agent: EventMachine 
Disallow: /

#For all crawlers
User-agent: * 
Disallow: /scripts/cder/bmis/
Disallow: /scripts/cder/BMIS/
Disallow: /scripts/cber/cfapps/
#Disallow: /scripts/cdrh/cfdocs/cfRL/     commented-out 13 Jul 2011 to allow crawl, by CN
Disallow: /scripts/email/
Disallow: /scripts/ImportRefusals/ir_index.cfm
Disallow: /scripts/warningletters/
Disallow: /scripts/cdrh/cfdocs/cfres/textresults.cfm
#Added devicesatfda, cftplc, search on 13 Jan 2014 by CN & LS
Disallow: /scripts/cdrh/devicesatfda/
Disallow: /scripts/cdrh/cfdocs/cftplc/
Disallow: /scripts/search/
#added CDRH excel disallow, 20 Jan 2014 by CN & LS
Disallow: /scripts/cdrh/*excel*.cfm
Disallow: /scripts/cdrh/*excel*.CFM
Disallow: /scripts/cdrh/*Excel*.cfm
Disallow: /scripts/cdrh/*Excel*.CFM
#added devicesatfda casing, 27 Feb 2014 by CN
Disallow: /SCRIPTs/cdrh/devicesatfda/

Hit-rate: 30 # wait 30 seconds before starting a new URL request default=30
Visiting-hours: 23:00EDT-05:00EDT #index this site between 11PM - 5AM EDT
Concurrent-hits: 2 # limit concurrent active URLS to 2 for each index server