# Engineering Leaders Community — robots.txt
# THIS FILE IS LIVE. Served by elc-web's engineeringleaders.io/* route since 2026-08-08, when the
# exact-path /robots.txt routes were removed from the elc-llms-txt Worker so there is one source
# of truth. That Worker still serves elc-conference.io/robots.txt and its llms.txt (Wix root,
# no other option) -- if you change the tier list, change it there too.
# Canonical template: /robots-txt skill. Edit tiers deliberately, not by deleting lines.
#
# Engineering Leaders Community — robots.txt
# https://www.engineeringleaders.io
# Canonical template: /robots-txt skill. Edit tiers deliberately, not by deleting lines.

# ── Tier 1: classic search ─────────────────────────────────────────
User-agent: Googlebot
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: Bingbot
Allow: /

User-agent: SeznamBot
Allow: /

User-agent: YandexBot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /

# ── Tier 2: AI retrieval bots -- these produce citations. A block here ─
# ── removes the site from that engine's answers, not just its index. ──
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Google-CloudVertexBot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: MistralAI-User
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# ── Tier 3: AI training bots -- allowed by default, see rationale above ─
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: CCBot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Timpibot
Allow: /

User-agent: Diffbot
Allow: /

# ── Tier 4: link unfurlers -- they fetch OG tags to build the preview card
# ── when someone shares a URL in Slack, LinkedIn or WhatsApp. Blocking one
# ── does not hide the link, it renders the share as a bare grey URL.
User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: Slackbot-LinkExpanding
Allow: /

User-agent: Discordbot
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: WhatsApp
Allow: /

# ── Tier 5: everyone else ──────────────────────────────────────────
User-agent: *
Content-Signal: ai-train=yes, search=yes, ai-input=yes
Allow: /

# Never-index intake form -- see CLAUDE.md: "never index that page"
Disallow: /newjoiner-queue/
Disallow: /partner/ceska-sporitelna/
Disallow: /partner/czechbusiness/
Disallow: /cs/partner/czechbusiness/
Disallow: /partner/navigara/plan/
Disallow: /patron/
Disallow: /cs/partner/cvut/

# Block Wix preview / staging artifacts
Disallow: /*?_escaped_fragment_=
Disallow: /*?wix-vod-video-id*
Disallow: /_partials/
Disallow: /_api/

# Single unified sitemap as of the 2026-08-04 Wix -> elc-web cutover. Before that, Wix served
# its own /sitemap.xml (root pages) and the standalone /cfp and /join Workers each shipped their
# own nested sitemap (Wix knew nothing about those, hence 3 separate lines here for a while) --
# both are gone now that everything lives in one Astro build with one @astrojs/sitemap output.
Sitemap: https://www.engineeringleaders.io/sitemap-index.xml

# Academy is a separately deployed subsite (its own Astro build, own Worker route at
# /academy*) not covered by the main sitemap above -- it ships its own sitemap-index.xml,
# which was previously undeclared here and therefore unreachable by crawlers (a robots.txt
# at /academy/robots.txt is not a valid location; only the root file is ever fetched).
Sitemap: https://www.engineeringleaders.io/academy/sitemap-index.xml

# AI agents: structured summary of ELC lives at https://www.engineeringleaders.io/llms.txt
# Deep companion, the full text of every meetup run to date, at
# https://www.engineeringleaders.io/llms-full.txt (generated by elc-web from
# src/content/events/*.md, not served by this Worker). This is an agent-tooling surface
# (Claude Code, MCP, coding assistants), not a search-citation lever -- see
# /ai-seo-llms-builder/references/evidence.md. Not listed as a Sitemap: above: it is markdown,
# not an XML sitemap, and Search Console reports it as an unreadable sitemap when declared that way.