## SLOW BINGBOT SO IT DOESN'T KEEP CRAWLING NON-EXISTENT PAGES
User-agent: Bingbot
Crawl-delay: 500
## BLOCK UNWANTED CRAWLERS
User-agent: MJ12bot
Disallow: /
User-agent: Yandex
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: spbot
Disallow: /
User-agent: Re-Animator
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-Agent: SMTBot
Disallow: /
## BLOCK SPECIFIC FOLDERS FROM BEING CRAWLED
User-agent: *
# Block digital media links (cause soft 404 in GWT)
Disallow: /flipper*
Disallow: /flipping-promotions*
# Block Wordpress folders
Disallow: /news/wp-admin/
#Disallow: /news/wp-content/
#Disallow: /news/wp-includes/
# Block non-existent pages from being crawled
Disallow: /b2c/*
Disallow: /category/000*
Disallow: /category*/www.*
Disallow: /category/*.php
Disallow: /category-products/000*
Disallow: /category-products/*.php
Disallow: /draper/*
Disallow: /en/*
Disallow: /mobile/
Disallow: /product/*.php
Disallow: /wordpress/*
Disallow: /b2cr*
Disallow: /*.html
Disallow: /*.shtml
Disallow: /cm-*
## BLOCK CRAWLERS INDEXING IMAGES & MANUALS
User-Agent: *
Disallow: /products
User-Agent: Googlebot
Allow: /products

User-agent: rogerbot
Allow: /products
User-agent: dotbot
Allow: /products

# Sitemaps
Sitemap: https://www.drapertools.com/sitemaps/categories-sitemap.xml
#Sitemap: http://www.drapertools.com/sitemaps/new-products-sitemap.xml
Sitemap: https://www.drapertools.com/sitemaps/products-sitemap.xml
Sitemap: https://www.drapertools.com/news/sitemap_index.xml