Sitemap: https://www.trespass.com/sitemap.xml

# Crawlers Setup

User-agent: *

# Directories
Disallow: /app/
Disallow: /bin/
Disallow: /directory/
Disallow: /downloader/
Disallow: /install/
Disallow: /js/
Disallow: /lib/
Disallow: /phpserver/
Disallow: /pkginfo/
Disallow: /private/
Disallow: /setup/
Disallow: /skin/
Disallow: /update/
Disallow: /var/
Disallow: /vendor/

# Paths (clean URLs)
Disallow: /admin/
Disallow: */api/
Disallow: */catalog/category/view/
Disallow: */catalog/product/view/
Disallow: */catalogsearch
Disallow: */checkout
Disallow: */contacts
Disallow: */customer/
Disallow: */newsletter
Disallow: */order/
Disallow: */poll/
Disallow: */report/
Disallow: */review/
Disallow: */rma/
Disallow: */sendfriend
Disallow: */storelocator/index/view/id/
Disallow: */wishlist

# Files
Disallow: /cron
Disallow: /cron.php

# Do not index pages that are sorted or filtered.
Disallow: */cat/
Disallow: /*/pricematch/
Disallow: /*/colour/
Disallow: /*/show/
Disallow: /*/size/
Disallow: */sort-by/
Disallow: /*?*product_list_dir=
Disallow: /*?*product_list_limit=
Disallow: /*?*product_list_mode=
Disallow: /*?*product_list_order=

# Clean URLs - Suffixes
Disallow: /*.php
Disallow: /*.sh$
#Disallow: /*?

# CVS, SVN directory and dump files
Disallow: /*.CSV$
Disallow: /*.csv$
Disallow: /*.gitignore$
Disallow: /*.sample$
Disallow: /*.sql$
Disallow: /*.zip$

# Parameters
Disallow: /*&carrier=
Disallow: /*&cat=
Disallow: /*&color_filter=
Disallow: /*&display_size=
Disallow: /*&name=
Disallow: /*&network=
Disallow: /*&os=
Disallow: /*&price=
Disallow: /*&timestamp=
Disallow: /*&top_seller=
Disallow: /*?SID=
Disallow: /*?___SID=
Disallow: /*?___store=
Disallow: /*?carrier=
Disallow: /*?cat=
Disallow: /*?color_filter=
Disallow: /*?display_size=
Disallow: /*?name=
Disallow: /*?network=
Disallow: /*?os=
# Disallow: /*?PageSpeed=noscript
Disallow: /*?price=
Disallow: /*?timestamp=
Disallow: /*?top_seller=

Disallow: /*?size_legacy=*
Disallow: /*?base_color=*
Disallow: /*?manufacturer=*
Disallow: /*?insulation_type=*
Disallow: /*?gender=*
Disallow: /*?p=2&size_legacy=*

Disallow: *?size_legacy=*
Disallow: *?base_color=*
Disallow: *?manufacturer=*
Disallow: *?insulation_type=*
Disallow: *?gender=*
Disallow: *?p=2&size_legacy=*

Disallow: /?adnetwork=
Disallow: /&adnetwork=
Disallow: /?affc=
Disallow: /&affc=
Disallow: /?cto_pld=
Disallow: /&cto_pld=

# Allow pagination
Allow: /*?p=

# Few bots obey the crawl delay but we will set it to 10
# Google ignores this so it does not matter that it is universally set

Crawl-Delay: 10

# Bad bots

User-agent: 008
Disallow: /

# Ahrefs is > 5% of traffic and can be disallowed on Black Friday. 20 seconds crawl delay specified for normal days
User-agent: AhrefsBot
Crawl-Delay: 20

# Amazon bot for Alexa does not respect crawl rate and is not behaving on N
User-agent: Amazonbot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: DotBot
Disallow: / 

User-agent: MJ12bot
Disallow: / 

User-agent: MauiBot
Disallow: / 

User-agent: MegaIndex.ru
Disallow: /

User-agent: Nutch
Disallow: / 

User-agent: PetalBot
Disallow: / 

User-agent: Seekbot
Disallow: /

User-agent: Sogou web spider
Disallow: /

User-agent: Sogou inst spider
Disallow: /

User-agent: Yandex
Disallow: / 

User-agent: Yandexbot
Disallow: / 

User-agent: adsbot
Disallow: /

User-agent: baiduspider
Disallow: / 

User-agent: baiduspider-image
Disallow: / 

User-agent: baiduspider-mobile
Disallow: / 

User-agent: baiduspider-news
Disallow: / 

User-agent: baiduspider-video
Disallow: / 

User-agent: barkrowler
Disallow: / 

user-agent: botify
Disallow: /

User-agent: ia_archiver
Disallow: / 

User-agent: idealo-bot
Disallow: /

User-agent: megaindex.com
Disallow: /

User-agent: newspaper/0.2.8
Disallow: /