# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
#
# To ban all spiders from the entire site uncomment the next two lines:
User-Agent: *
Disallow: /subscribe/*
Disallow: /wiki/revert/*
Disallow: /stats/*
Disallow: /api/*
Host: publiclab.org

# limit to 1 per day
User-agent: AhrefsBot 
Crawl-Delay: 86400
