# robots.txt — https://www.express.com
# Updated: 2026-05-27
# ======================================================================
# GROUP 1 — Default
# Applies to all crawlers not specified in GROUP 2
# ======================================================================
User-agent: *
Disallow: /blog/wp-admin/
Disallow: /checkout/
Disallow: /check-out/
Disallow: /user/
Disallow: /account/
Disallow: /catalog/gadgets/
Disallow: /browse/gadgets/
Disallow: /coupons/
Disallow: /mobile/
Disallow: /clothing/*Dump*
Disallow: /clothing/*dump*
Disallow: /clothing/*Root*
Disallow: /clothing/*root*
Disallow: /content/express/
Disallow: /login?*
Disallow: /browse/page.jsp?page=/browse/gadgets/gift-card-balance.jsp
Disallow: /browse/actions/twitter-share-submit.jsp

# The site is client-side rendered; /graphql feeds ALL page content.
Allow: /graphql

Crawl-delay: 2

# ======================================================================
# GROUP 2 — Named search + AI crawlers (explicitly allowed)
# ======================================================================

# --- Traditional search ---
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Bingbot

# --- AI answer engines / live retrieval (can cite you and send traffic) ---
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: DuckAssistBot
User-agent: MistralAI-User
User-agent: Amazonbot
User-agent: Applebot

# --- AI model-training crawlers (allowed; set "Disallow: /" to opt out) ---
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: CCBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent

Disallow: /blog/wp-admin/
Disallow: /checkout/
Disallow: /check-out/
Disallow: /user/
Disallow: /account/
Disallow: /catalog/gadgets/
Disallow: /browse/gadgets/
Disallow: /coupons/
Disallow: /mobile/
Disallow: /clothing/*Dump*
Disallow: /clothing/*dump*
Disallow: /clothing/*Root*
Disallow: /clothing/*root*
Disallow: /content/express/
Disallow: /login?*
Disallow: /browse/page.jsp?page=/browse/gadgets/gift-card-balance.jsp
Disallow: /browse/actions/twitter-share-submit.jsp
Allow: /graphql
Crawl-delay: 2

# ======================================================================
# Sitemap — global directive, independent of the groups above
# ======================================================================
Sitemap: https://www.express.com/siteindex.xml