# MovieFarming Robots.txt — 2026 SEO & GEO Edition
# https://www.moviefarming.com/robots.txt

# ─── Content Signals (RFC draft-romm-aipref-contentsignals) ─────────────
# Declares AI content usage preferences for crawlers and agents.
Content-Signal: ai-train=accepted, search=yes, ai-input=no
Content-Signal: training=accepted, indexing=allowed

# ─── Major Search Engine Crawlers ──────────────────────────────────────

User-agent: Googlebot
User-agent: Bingbot
User-agent: DuckDuckBot
Allow: /
Disallow: /api/
Disallow: /_next/
Disallow: /private/
Crawl-delay: 1

# ─── Apple & Social Platform Crawlers ──────────────────────────────────

User-agent: Applebot
User-agent: FacebookBot
Allow: /
Disallow: /api/
Disallow: /private/

# ─── AI Answer Engine Crawlers (GEO / AEO Citations) ───────────────────
# These crawlers power AI Overviews, ChatGPT Search, Perplexity, Gemini.
# Allowing them enables your content to be cited in AI-generated answers.

User-agent: GPTBot
User-agent: ChatGPT-User
Allow: /movie/
Allow: /tv/
Allow: /actor/
Allow: /director/
Allow: /discover
Allow: /trending
Allow: /genre/
Allow: /year/
Allow: /reviews/
Allow: /ending-explained/
Allow: /recommendation/
Allow: /
Disallow: /api/
Disallow: /private/

User-agent: PerplexityBot
User-agent: Omgilibot
Allow: /movie/
Allow: /tv/
Allow: /actor/
Allow: /director/
Allow: /discover
Allow: /trending
Allow: /genre/
Allow: /year/
Allow: /reviews/
Allow: /ending-explained/
Allow: /recommendation/
Allow: /
Disallow: /api/
Disallow: /private/

User-agent: ClaudeBot
Allow: /movie/
Allow: /tv/
Allow: /actor/
Allow: /director/
Allow: /discover
Allow: /trending
Allow: /genre/
Allow: /year/
Allow: /reviews/
Allow: /ending-explained/
Allow: /recommendation/
Allow: /
Disallow: /api/
Disallow: /private/

User-agent: OAI-SearchBot
Allow: /movie/
Allow: /tv/
Allow: /actor/
Allow: /director/
Allow: /discover
Allow: /trending
Allow: /
Disallow: /api/
Disallow: /private/

# Google-Extended — Google's AI training crawler
User-agent: Google-Extended
Allow: /movie/
Allow: /tv/
Allow: /actor/
Allow: /director/
Allow: /discover
Allow: /trending
Allow: /
Disallow: /api/
Disallow: /images/

# ─── Block Aggressive SEO Scrapers ─────────────────────────────────────
# These bots consume crawl budget without providing search/AI value.

User-agent: AhrefsBot
Disallow: /api/
Crawl-delay: 10

User-agent: MJ12bot
Disallow: /

User-agent: SemrushBot
Disallow: /api/
Crawl-delay: 10

User-agent: DotBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: TikTokBot
Disallow: /

# ─── Standard Restrictions (applies to all other crawlers) ─────────────

User-agent: *
Allow: /
Disallow: /private/
Disallow: /api/
Disallow: /_next/
Disallow: /profile
Disallow: /history
Disallow: /watchlist
Disallow: /cheats
Disallow: /oracle
Disallow: /login
Disallow: /signup
Crawl-delay: 1

# ─── Sitemaps ─────────────────────────────────────────────────────────
Sitemap: https://www.moviefarming.com/sitemap.xml
Sitemap: https://www.moviefarming.com/sitemap-movies.xml
Sitemap: https://www.moviefarming.com/sitemap-tv.xml
Sitemap: https://www.moviefarming.com/sitemap-actors.xml
Sitemap: https://www.moviefarming.com/sitemap-images.xml
Sitemap: https://www.moviefarming.com/sitemap-video.xml
Sitemap: https://www.moviefarming.com/sitemap-news.xml