# DomainDrift -- Continuous Internet Telemetry Index
# https://domaindrift.io

User-agent: *
Allow: /

# THE JSON API IS THE PRODUCT for agents, and the agents that respect robots.txt (the ones
# Anthropic and OpenAI ship) are exactly the ones we want to reach it. A Disallow here does NOT
# protect the search index - the API is linked from no page and listed in no sitemap, so a crawler
# has no path to it either way - it only turns away the well-behaved agent fetchers this product is
# built for, while the badly-behaved ones ignore robots and sail through. So the agent-facing
# surface is EXPLICITLY allowed (the keyless curl in our own hero is /v1/domains/:d):
Allow: /v1/
# /v1/ above is the canonical API base. The older base below is a permanent alias: it is
# frozen inside receipts that were already signed and published, so those URLs have to keep
# resolving - and therefore keep being allowed - forever.
Allow: /connor/v1/
Allow: /mcp
Allow: /.well-known/
Allow: /receipts/
Allow: /llms.txt
Allow: /llms-full.txt

# Session plumbing - nothing to index, and nothing an agent needs.
Disallow: /login
Disallow: /logout
Disallow: /sso/

# NOTE, deliberately: the sign-in-gated pages (/app, /domains, /changes, /services, /certs,
# /browse, /domain/*, /t/*/<timestamp>, /evidence/*, /bundle/*) are NOT disallowed. They serve
# "noindex" and appear in no sitemap. A Disallow would prevent the crawler from ever READING
# that noindex, and Google could still index a bare URL from an inbound link. Crawlable +
# noindex is what actually keeps a paywalled page out of the index. Every one of those pages
# serves a signed-out visitor and Googlebot the exact same bytes - there is no crawler branch
# in this codebase, because a special case for the bot is cloaking.

Sitemap: https://domaindrift.io/sitemap.xml
Sitemap: https://domaindrift.io/sitemap-domains.xml
Sitemap: https://domaindrift.io/sitemap-providers.xml