User-agent: *
Disallow: /default.aspx?q=*
Allow: /default.aspx?stores*
Disallow: /default.aspx?*action=*
Disallow: /default.aspx?Z
Disallow: /default.aspx?z
Disallow: /default.aspx?action=forms
Disallow: *?filter*
Disallow: *filter&aspect*
Disallow: *All%20Items&aspect*
Disallow: /search?q*
Disallow: *refine=*
Disallow: *sort=*

# AdsBot-Google has to be blocked explicitly (https://support.google.com/adwords/answer/2404197?hl=en)
User-agent: AdsBot-Google
Allow: /default.aspx?q=*
Allow: /default.aspx?stores*
Disallow: /default.aspx?*action=*
Disallow: /default.aspx?Z
Disallow: /default.aspx?z
Disallow: /default.aspx?action=forms
Disallow: *?filter*
Disallow: *filter&aspect*
Disallow: *All%20Items&aspect*
Disallow: /search?q*
Disallow: *refine=*
Disallow: *sort=*

Sitemap: https://www.davidjones.com/sitemap.xml
Sitemap: https://www.davidjones.com/pdp-sitemap.xml
Sitemap: https://www.davidjones.com/blog/sitemap.xml

# Implement 10s crawl delay for SEO tools
User-agent: AhrefsSiteAudit
User-agent: AhrefsBot
User-agent: MJ12bot
Disallow: /default.aspx?q=*
Allow: /default.aspx?stores*
Disallow: /default.aspx?*action=*
Disallow: /default.aspx?Z
Disallow: /default.aspx?z
Disallow: /default.aspx?action=forms
Disallow: *?filter*
Disallow: *filter&aspect*
Disallow: *All%20Items&aspect*
Disallow: /search?q*
Disallow: *refine=*
Disallow: *sort=*
Crawl-delay: 10

# Implement 1s crawl delay for search engines not widely used in Australia
User-agent: Yandexbot
User-agent: Baiduspider
User-agent: SeznamBot
User-agent: Slurp
User-agent: Pinterestbot
Disallow: /default.aspx?q=*
Allow: /default.aspx?stores*
Disallow: /default.aspx?*action=*
Disallow: /default.aspx?Z
Disallow: /default.aspx?z
Disallow: /default.aspx?action=forms
Disallow: *?filter*
Disallow: *filter&aspect*
Disallow: *All%20Items&aspect*
Disallow: /search?q*
Disallow: *refine=*
Disallow: *sort=*
Crawl-delay: 1