# For all robots
User-agent: *

# Block access to specific groups of pages

Disallow: /uk/bag
Disallow: /uk/checkout
Disallow: /uk/my-account
Disallow: /uk/account

Disallow: /us/bag
Disallow: /us/checkout
Disallow: /us/my-account
Disallow: /us/account

# Allow search crawlers to discover the sitemap
Sitemap: https://www.thewhitecompany.com/uk/uk-sitemap.xml
Sitemap: https://www.thewhitecompany.com/us/us-sitemap.xml

Disallow: /quickView

Disallow: /*?page=
Allow: /*?page=1$
Allow: /*?page=2$
Allow: /*?page=3$
Allow: /*?page=4$
Allow: /*?page=5$
Allow: /*?page=6$
Allow: /*?page=7$
Allow: /*?page=8$
Allow: /*?page=9$
Allow: /*?page=10$
Allow: /*?page=11$
Allow: /*?page=12$
Allow: /*?page=13$
Allow: /*?page=14$
Allow: /*?page=15$
Allow: /*?page=16$
Allow: /*?page=17$
Allow: /*?page=18$
Allow: /*?page=19$

# Remove duplication caused by URL facets.
Disallow: *?q=

# Block CazoodleBot as it does not present correct accept content headers
User-agent: CazoodleBot
Disallow: /

# Block MJ12bot as it is just noise
User-agent: MJ12bot
Disallow: /

# Block dotbot as it cannot parse base urls properly
User-agent: dotbot/1.0
Disallow: /

# Block Gigabot
User-agent: Gigabot
Disallow: /

# Block PDF
Disallow: *.pdf$