# robots.txt for autodata2016 (www.autodata.nl)

# autodiscovery sitemaps
Sitemap: https://www.autodata.nl/sitemap.xml
Sitemap: https://www.autodata.nl/sitemap_occasions.xml

User-agent: *
Disallow: /help/
Disallow: /releases
Disallow: /cgi/comments.cgi
Disallow: /cgi/ecard.cgi
Disallow: /cgi/formmail
Disallow: /cgi/login.cgi
Disallow: /cgi/register.cgi
Disallow: /cgi/search.cgi
Disallow: */empty

User-agent: xcms_search_engine
Disallow: 
Allow: /releases

User-agent: *
Crawl-delay: 5

User-agent: Googlebot-Image
Disallow: /*static-storage/

# ongebruikelijke robots
User-agent: webcrawler
Disallow: /

User-agent: XoviBot
Disallow: /

User-agent: BLEXBot
Disallow: /

user-agent: AhrefsBot
disallow: /

User-agent: Exabot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: trovitBot
Disallow: /

User-agent: Aboundexbot
Disallow: / 

User-agent: BLEXBot
Disallow: /

User-agent: KingKevinBot
Disallow: /