# Production Robots.txt file

# Sitemap
Sitemap: https://www.shopstyle.com.au/sitemap-index.https.xml

User-agent: YandexBot
Crawl-delay: 10

# Baidu doesnt support Crawl-delay but added anyways in case they ever do
User-agent: Baiduspider
Crawl-delay: 10

User-agent: EtaoSpider
Disallow: /

User-agent: Facebot
Disallow:

User-agent: Twitterbot
Disallow:

User-agent: *
Disallow: /util/request-headers
Disallow: /*wenhao*
Disallow: /trend/
Disallow: /action/
Disallow: /search
Disallow: /*crawl=no*
Disallow: /users
Disallow: /lists
Disallow: /layouts
Disallow: /track
Disallow: /collective/
Disallow: /featured-looks
Disallow: /p/
Disallow: /l/