# robots.txt for https://najga.org/
# Last updated: [July 1, 2026]
#
# Strategy: Allow anything that cites NAJGA back (search engines and
# citation-based AI assistants). Block bulk-harvest training crawlers that
# take content without attribution. The Terms of Use is the legal backstop;
# robots.txt is the honor-system signal; Cloudflare AI Crawl Control is the
# edge enforcement. All three should stay aligned: any user-agent blocked at
# the Cloudflare edge should also be Disallowed here.

# ===== TRADITIONAL SEARCH ENGINES: ALLOW =====
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /

# Baidu search (China's dominant search engine; sends referral traffic,
# cites back). Added 7/1/26 after Baiduspider appeared in AI Crawl Control logs.
User-agent: Baiduspider
Allow: /

# ===== AI SEARCH AND CITATION-BASED ASSISTANTS: ALLOW =====
# These bots retrieve pages when a user asks an AI assistant a question,
# and they surface NAJGA as a cited source in the answer.
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: DuckAssistBot
Allow: /

# ===== AI TRAINING CRAWLERS: BLOCK =====
# These collect bulk content for model training and do not send traffic back.
# The Terms of Use prohibits AI training use of NAJGA content regardless of
# whether a crawler respects these directives.

# OpenAI training crawler
User-agent: GPTBot
Disallow: /

# Anthropic training crawler
User-agent: ClaudeBot
Disallow: /

# Anthropic legacy user agents (deprecated but included for completeness)
User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

# Google Gemini / Vertex TRAINING and grounding opt-out. Disallowed 6/30/26.
# Note: this does NOT affect Google Search ranking or AI Overviews (those run
# on Googlebot, allowed above). Blocking it only declines Gemini training use,
# consistent with the block-training stance.
User-agent: Google-Extended
Disallow: /

# Apple Intelligence training (Apple AI does not cite sources back)
User-agent: Applebot-Extended
Disallow: /

# Meta AI training crawlers. Meta does not cite sources back.
User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Meta-ExternalFetcher
Disallow: /

User-agent: FacebookBot
Disallow: /

# Amazon (Alexa, Amazon AI features; does not cite sources)
User-agent: Amazonbot
Disallow: /

# ByteDance (TikTok parent). Known for aggressive crawling.
User-agent: Bytespider
Disallow: /

# Common Crawl. Feeds a broad open corpus reused by many AI developers.
User-agent: CCBot
Disallow: /

# Cohere training crawler
User-agent: cohere-ai
Disallow: /

# Huawei Petal crawler (Petal search + AI). Added 6/30/26 to match the
# Cloudflare edge block.
User-agent: PetalBot
Disallow: /

# Google Vertex owner-requested crawler. Added 6/30/26 to match the edge block.
User-agent: Google-CloudVertexBot
Disallow: /

# ProRata AI crawler. Added 6/30/26 to match the edge block.
User-agent: ProRataInc
Disallow: /

# Timpi crawler. Added 6/30/26 to match the edge block.
User-agent: Timpibot
Disallow: /

# Data extraction tools
User-agent: Diffbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Omgili
Disallow: /

# ===== SITEMAP =====
Sitemap: https://najga.org/sitemap_index.xml