# Endless Travel Plans - robots.txt
# https://www.endlesstravelplans.com

User-agent: *
# Auth callbacks and JSON helpers are not content (GSC crawled them, 2026-09-16)
Disallow: /auth/
Disallow: /csrf-token
Disallow: /ingest/

# ========================================
# Block Old WordPress URLs (Added 2026-01-06)
# These pages were permanently removed during WordPress → Laravel migration
# ========================================
Disallow: /destinations/by-travel-style/
Disallow: /destinations/by-season/
Disallow: /plan-your-trip/
Disallow: /my-trips/
Disallow: /test_*
Disallow: /wp-admin/
Disallow: /wp-content/
Disallow: /wp-includes/

# Sitemap location
Sitemap: https://www.endlesstravelplans.com/sitemap.xml

# ========================================
# AI Crawler Permissions
# Explicitly allow AI training and citation crawlers
# ========================================

# OpenAI - training
User-agent: GPTBot
Allow: /
Crawl-delay: 2

# OpenAI - live retrieval (added v3)
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic - training
User-agent: ClaudeBot
Allow: /
Crawl-delay: 2

User-agent: anthropic-ai
Allow: /

# Anthropic - live retrieval (added v3)
User-agent: Claude-SearchBot
Allow: /

# Perplexity - training
User-agent: PerplexityBot
Allow: /

# Perplexity - live retrieval (added v3)
User-agent: Perplexity-User
Allow: /

# Common Crawl (used by many LLMs)
User-agent: CCBot
Allow: /
Crawl-delay: 2

# Google AI
User-agent: Google-Extended
Allow: /

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /
Crawl-delay: 2

# Amazon
User-agent: Amazonbot
Allow: /

# Microsoft/Bing
User-agent: Bingbot
Allow: /

# ========================================
# Block SEO Competitor Tools (Added v3 2026-04-27)
# Consume bandwidth without citation upside
# ========================================
User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: BLEXBot
Disallow: /

# Reference to AI-specific discovery file
# llms.txt: https://www.endlesstravelplans.com/llms.txt