# robots.txt for https://www.halliwelljones.co.uk/ sitemap: https://www.halliwelljones.co.uk/sitemaps-1-sitemap.xml # live - allow search engines and SemrushBot, block others # Allow major search engines User-agent: Googlebot Disallow: /cpresources/ Disallow: /vendor/ Disallow: /.env Disallow: /cache/ Disallow: /search Disallow: /vehicles Disallow: /actions/ User-agent: Bingbot Disallow: /cpresources/ Disallow: /vendor/ Disallow: /.env Disallow: /cache/ Disallow: /search Disallow: /vehicles Disallow: /actions/ User-agent: Slurp Disallow: /cpresources/ Disallow: /vendor/ Disallow: /.env Disallow: /cache/ Disallow: /search Disallow: /vehicles Disallow: /actions/ User-agent: DuckDuckBot Disallow: /cpresources/ Disallow: /vendor/ Disallow: /.env Disallow: /cache/ Disallow: /search Disallow: /vehicles Disallow: /actions/ User-agent: Baiduspider Disallow: /cpresources/ Disallow: /vendor/ Disallow: /.env Disallow: /cache/ Disallow: /search Disallow: /vehicles Disallow: /actions/ User-agent: Yandex Disallow: /cpresources/ Disallow: /vendor/ Disallow: /.env Disallow: /cache/ Disallow: /search Disallow: /vehicles Disallow: /actions/ # Allow Semrush User-agent: SemrushBot Disallow: /cpresources/ Disallow: /vendor/ Disallow: /.env Disallow: /cache/ Disallow: /search Disallow: /vehicles Disallow: /actions/ # Block all other SEO crawlers User-agent: AhrefsBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: DotBot Disallow: / User-agent: BLEXBot Disallow: / User-agent: PetalBot Disallow: / User-agent: Sogou Disallow: / User-agent: MegaIndex Disallow: / User-agent: SEOkicks Disallow: / User-agent: seoscanners.net Disallow: / User-agent: SEOBility Disallow: / User-agent: SiteAuditBot Disallow: / User-agent: * Disallow: /cpresources/ Disallow: /vendor/ Disallow: /.env Disallow: /cache/ Disallow: /search Disallow: /vehicles Disallow: /actions/ Disallow: /forms/modal-* Disallow: /404