# DeskFitPro robots.txt
# Updated: 2026-08-25

# Default: allow all crawlers
User-agent: *
Allow: /

# ============================================
# AI Search Crawlers — ALLOWED (send traffic)
# ============================================

# Google Search + AI Overviews
User-agent: Googlebot
Allow: /

# Bing Search + Copilot
User-agent: Bingbot
Allow: /

# Perplexity AI (cites sources with links)
User-agent: PerplexityBot
Allow: /

# Apple Intelligence / Siri
User-agent: Applebot
Allow: /

# Yandex Search
User-agent: YandexBot
Allow: /

# ChatGPT browsing agent (cites sources when browsing)
User-agent: ChatGPT-User
Allow: /

# Claude / Anthropic (cites sources)
User-agent: ClaudeBot
Allow: /

# Cohere (powers search features)
User-agent: cohere-ai
Allow: /

# Facebook/Meta link previews
User-agent: FacebookExternalHit
Allow: /

# ============================================
# Training-Only Scrapers — BLOCKED
# (bulk scraping, no traffic or attribution)
# ============================================

# Common Crawl — bulk scraping, no traffic back
User-agent: CCBot
Disallow: /

# OpenAI training crawler (NOT the browsing agent)
User-agent: GPTBot
Disallow: /

# Google AI training (NOT Googlebot search)
User-agent: Google-Extended
Disallow: /

# ByteDance/TikTok scraper
User-agent: Bytespider
Disallow: /

# Amazon scraper
User-agent: Amazonbot
Disallow: /

# Anthropic training (separate from ClaudeBot)
User-agent: anthropic-ai
Disallow: /

# AI training scrapers
User-agent: Omgilibot
Disallow: /

User-agent: Diffbot
Disallow: /

# Sitemap (legacy WordPress sitemap paths permanently redirect here)
Sitemap: https://deskfitpro.com/sitemap-index.xml

# LLM / AEO site index (https://llmstxt.org)
# https://deskfitpro.com/llms.txt