# Robots.txt for Federation Trade Compliance Platform
# Production: https://federationlabs.ai
# Last updated: 2026-04-10
#
# SEO Strategy: Main product pages (landing, about, blog)
#                are fully crawlable with enhanced GA4 tracking. Analysis features
#                have been consolidated into the workspace app (app.federationlabs.ai).
#                API endpoints and internal tools are blocked to focus crawl budget
#                on high-value content.
#
# GEO Strategy: AI crawlers (GPTBot, ClaudeBot, PerplexityBot, Google-Extended,
#                OAI-SearchBot, Applebot-Extended, etc.) are explicitly allowed on
#                public product pages so Federation can be cited in ChatGPT,
#                Perplexity, Gemini, and Claude responses. B2B SaaS visibility
#                trade-off: we prefer AI citation reach over blocking training.
#                See llms.txt for the canonical content map.

# ==============================================================================
# Default crawling rules for all search engines
# ==============================================================================
User-agent: *

# Allow landing pages (all languages)
Allow: /$
Allow: /en$
Allow: /ko$
Allow: /en/$
Allow: /ko/$

# Allow main product pages
Allow: /en/about$
Allow: /ko/about$
Allow: /en/contact$
Allow: /ko/contact$
Allow: /en/blog
Allow: /ko/blog
Allow: /en/hs-code-classification$
Allow: /ko/hs-code-classification$
Allow: /en/tariff-fta-analysis$
Allow: /ko/tariff-fta-analysis$
Allow: /en/regulatory-compliance$
Allow: /ko/regulatory-compliance$
# Allow static assets (CSS, JS, images, fonts)
Allow: /assets/
Allow: /landing/assets/
Allow: /about/assets/
Allow: /blog/assets/
Allow: /static/blog/
Allow: /*.js$
Allow: /*.css$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.svg$
Allow: /*.webp$
Allow: /*.woff$
Allow: /*.woff2$
Allow: /*.ico$

# Block API endpoints (preserve server resources)
Disallow: /api/

# Block workspace (requires authentication)
Disallow: /workspace/

# Block medevice (moved to signalblue.ai)
Disallow: /en/medevice
Disallow: /ko/medevice

# Block temporary/internal paths
Disallow: /admin/
Disallow: /private/
Disallow: /__/

# Block query parameters and configuration files
Disallow: /*?*
Disallow: /*.json$
Disallow: /*.xml$
Allow: /sitemap.xml$
Allow: /robots.txt$

Crawl-delay: 1

# ==============================================================================
# Google bots - Inherit default rules
# ==============================================================================
User-agent: Googlebot
# Uses default rules from User-agent: *

User-agent: Googlebot-Image
Allow: /assets/
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.svg$
Allow: /*.webp$
Allow: /*.ico$
Disallow: /api/

# ==============================================================================
# Bing bot - Microsoft search engine
# ==============================================================================
User-agent: Bingbot
# Uses default rules from User-agent: *
Crawl-delay: 1

# ==============================================================================
# Other major search engines - Inherit default rules
# ==============================================================================

# Yahoo (Slurp)
User-agent: Slurp
# Uses default rules from User-agent: *

# DuckDuckGo
User-agent: DuckDuckBot
# Uses default rules from User-agent: *

# Baidu (Chinese search engine - limited access)
User-agent: Baiduspider
Allow: /$
Allow: /en$
Allow: /ko$
Allow: /assets/
Disallow: /api/
Crawl-delay: 2

# Yandex (Russian search engine)
User-agent: Yandex
# Uses default rules from User-agent: *

# Naver (Korean search engine - prioritize Korean content)
User-agent: NaverBot
Allow: /$
Allow: /ko$
Allow: /ko/
Allow: /assets/
Allow: /ko/about$
Allow: /ko/blog
Allow: /ko/hs-code-classification$
Allow: /ko/tariff-fta-analysis$
Allow: /ko/regulatory-compliance$
Allow: /static/blog/
Disallow: /api/
Disallow: /workspace/

# ==============================================================================
# SEO Tools - Rate-limited access for monitoring and analytics
# ==============================================================================

# Ahrefs (SEO monitoring)
User-agent: AhrefsBot
Crawl-delay: 10
# Uses default rules with rate limiting

# Semrush (SEO analytics)
User-agent: SemrushBot
Crawl-delay: 10
# Uses default rules with rate limiting

# Moz (SEO metrics)
User-agent: DotBot
Crawl-delay: 10
# Uses default rules with rate limiting

# Majestic (backlink analysis)
User-agent: MJ12bot
Crawl-delay: 10
# Uses default rules with rate limiting

# ==============================================================================
# Block aggressive, unwanted, or problematic bots
# ==============================================================================

# Huawei cloud bot (aggressive crawling)
User-agent: PetalBot
Disallow: /

# Built With (excessive crawling)
User-agent: BLEXBot
Disallow: /

# DataForSEO (commercial data mining)
User-agent: DataForSeoBot
Disallow: /

# MegaIndex (aggressive Russian bot)
User-agent: MegaIndex
Disallow: /

# Seznam (Czech search engine - rate limited)
User-agent: SeznamBot
Crawl-delay: 10

# ==============================================================================
# AI crawlers - Explicitly allowed for GEO (Generative Engine Optimization)
# These bots feed ChatGPT, Perplexity, Gemini, Claude, Copilot citations.
# Public product pages + blog are crawlable; API + workspace stay blocked.
# ==============================================================================

# OpenAI - ChatGPT training (model improvement)
User-agent: GPTBot
Allow: /en/
Allow: /ko/
Allow: /static/
Allow: /assets/
Allow: /llms.txt$
Disallow: /api/
Disallow: /workspace/
Crawl-delay: 2

# OpenAI - ChatGPT search index (SearchGPT / ChatGPT Search)
User-agent: OAI-SearchBot
Allow: /en/
Allow: /ko/
Allow: /static/
Allow: /assets/
Allow: /llms.txt$
Disallow: /api/
Disallow: /workspace/
Crawl-delay: 2

# OpenAI - ChatGPT user-initiated browsing
User-agent: ChatGPT-User
Allow: /en/
Allow: /ko/
Allow: /static/
Allow: /assets/
Allow: /llms.txt$
Disallow: /api/
Disallow: /workspace/

# Anthropic - Claude training
User-agent: anthropic-ai
Allow: /en/
Allow: /ko/
Allow: /static/
Allow: /assets/
Allow: /llms.txt$
Disallow: /api/
Disallow: /workspace/
Crawl-delay: 2

# Anthropic - Claude search / retrieval
User-agent: ClaudeBot
Allow: /en/
Allow: /ko/
Allow: /static/
Allow: /assets/
Allow: /llms.txt$
Disallow: /api/
Disallow: /workspace/
Crawl-delay: 2

# Anthropic - Claude user-initiated browsing
User-agent: Claude-Web
Allow: /en/
Allow: /ko/
Allow: /static/
Allow: /assets/
Allow: /llms.txt$
Disallow: /api/
Disallow: /workspace/

# Perplexity - search and citation
User-agent: PerplexityBot
Allow: /en/
Allow: /ko/
Allow: /static/
Allow: /assets/
Allow: /llms.txt$
Disallow: /api/
Disallow: /workspace/
Crawl-delay: 2

# Perplexity - user-initiated actions
User-agent: Perplexity-User
Allow: /en/
Allow: /ko/
Allow: /static/
Allow: /assets/
Allow: /llms.txt$
Disallow: /api/
Disallow: /workspace/

# Google - Gemini / AI Overviews (separate from Googlebot)
User-agent: Google-Extended
Allow: /en/
Allow: /ko/
Allow: /static/
Allow: /assets/
Allow: /llms.txt$
Disallow: /api/
Disallow: /workspace/

# Apple - Apple Intelligence
User-agent: Applebot-Extended
Allow: /en/
Allow: /ko/
Allow: /static/
Allow: /assets/
Allow: /llms.txt$
Disallow: /api/
Disallow: /workspace/

# Common Crawl - feeds many LLM training pipelines
User-agent: CCBot
Allow: /en/
Allow: /ko/
Allow: /static/
Allow: /assets/
Allow: /llms.txt$
Disallow: /api/
Disallow: /workspace/
Crawl-delay: 5

# Meta / Facebook - AI training
User-agent: FacebookBot
Allow: /en/
Allow: /ko/
Allow: /static/
Allow: /assets/
Disallow: /api/
Disallow: /workspace/

User-agent: Meta-ExternalAgent
Allow: /en/
Allow: /ko/
Allow: /static/
Allow: /assets/
Allow: /llms.txt$
Disallow: /api/
Disallow: /workspace/

# Cohere - LLM retrieval
User-agent: cohere-ai
Allow: /en/
Allow: /ko/
Allow: /static/
Allow: /assets/
Allow: /llms.txt$
Disallow: /api/
Disallow: /workspace/

# Bytespider - ByteDance / Doubao (block aggressive crawl)
User-agent: Bytespider
Disallow: /

# ==============================================================================
# Sitemap and LLMs.txt location
# ==============================================================================
Sitemap: https://federationlabs.ai/sitemap.xml