# Allow all major bots full access, except media files: keeps thumbnails, trailers and
# member files out of image search (UK restrictions, 2026-10-29). Pages still index;
# Twitterbot and facebookexternalhit follow their own groups below, so link cards keep their images.
User-agent: *
Disallow: /content/
Disallow: /members/
Disallow: /upload/

# Explicit allow for Twitterbot
User-agent: Twitterbot
Allow: /

# Explicit allow for Facebook crawler
User-agent: facebookexternalhit
Allow: /

# Block common non-content areas (optional, for SEO hygiene)
Disallow: /login/
Disallow: /register/
Disallow: /join/
Disallow: /cart/
Disallow: /checkout/
Disallow: /account/

# Crawl delay for aggressive bots (optional)
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

# Sitemap