# Machine-readable descriptions of this site live at:
#   /llms.txt                    what Diali is, in prose and links (llmstxt.org)
#   /llms.md                     the same document, for clients that probe the .md extension
#   /index.md                    the site root as markdown, for an agent arriving cold
#   /openapi.json                the public API, as OpenAPI 3.1
#   /auth.md                     what a credential does and does not get you (workos/auth.md)
#   /pricing.md                  plans and prices, rendered from the live catalog
#   /faq.md                      every FAQ question and answer, flattened
#   /developers.md               the developer guide: endpoints, units, errors, examples
#   /developers/llms.txt         the same, section-scoped, for an agent that only needs the API
#   /blog/llms.txt               every post with its date and summary, scoped to the writing
#   /channels/llms.txt           the messaging channels an assistant can be reached on
#   /schemamap.xml               NLWeb Schema Map: where the bulk structured-data feeds are
#   /feeds/blog.jsonl            every post as schema.org BlogPosting, one per line
#   /feeds/faq.jsonl             the FAQ as a schema.org FAQPage
#   /.well-known/apis.json       API discovery index (apisjson.org)
#   /.well-known/api-catalog     API catalog linkset (RFC 9727)
#   /.well-known/ard.json        Agentic Resource Discovery catalog
#   /.well-known/agent-skills/index.json   Agent Skills discovery index
#   /.well-known/security.txt    how to report a vulnerability
#
# Any page with a markdown twin also answers `Accept: text/markdown` on its own URL, with
# `Vary: Accept`. Appending `.md` to the path does the same thing without a header.

User-Agent: *
# The catalog is public, unauthenticated product data (plans, prices, engines, channels, models,
# skills) that llms.txt and /openapi.json both point agents at, so it is carved out of the blanket
# /api/ block. Longest-match wins, so this Allow beats the Disallow below for these paths only.
Allow: /api/v1/catalog/
Disallow: /api/
Disallow: /dashboard/
Disallow: /fr/api/
Disallow: /fr/dashboard/
Disallow: /es/api/
Disallow: /es/dashboard/
Disallow: /de/api/
Disallow: /de/dashboard/
Disallow: /pt/api/
Disallow: /pt/dashboard/
Disallow: /it/api/
Disallow: /it/dashboard/
Disallow: /id/api/
Disallow: /id/dashboard/
Disallow: /ja/api/
Disallow: /ja/dashboard/
Disallow: /ar/api/
Disallow: /ar/dashboard/

# ─────────────────────────────────────────────────────────────────────────────────────────────
# AI crawlers and assistants: welcome, on the same terms as everyone else.
#
# READ THIS BEFORE EDITING. Under RFC 9309 a crawler obeys exactly ONE group — the most specific
# one matching its token — and ignores every other group, `*` included. So this group is not an
# addition to the rules above; for these agents it REPLACES them, and the Disallow lines have to
# be repeated verbatim. Dropping them here would open /dashboard/ and the session-gated API to
# precisely the clients most likely to fetch every URL they are handed.
#
# `Allow: /` is redundant against a default-allow protocol and is stated anyway: it is the only
# unambiguous way to say "yes, deliberately" to an agent (or an auditor) reading this file, rather
# than leaving consent to be inferred from silence.
#
# The list is a curated allowlist of identifiable, documented agents, not an exhaustive one. An
# agent absent from it falls through to `*` above and is still allowed — being named here buys an
# explicit welcome, nothing more.
# ─────────────────────────────────────────────────────────────────────────────────────────────

User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: MistralAI-User
User-agent: DuckAssistBot
User-agent: Amazonbot
User-agent: cohere-ai
User-agent: CCBot
Allow: /
Allow: /api/v1/catalog/
Disallow: /api/
Disallow: /dashboard/
Disallow: /fr/api/
Disallow: /fr/dashboard/
Disallow: /es/api/
Disallow: /es/dashboard/
Disallow: /de/api/
Disallow: /de/dashboard/
Disallow: /pt/api/
Disallow: /pt/dashboard/
Disallow: /it/api/
Disallow: /it/dashboard/
Disallow: /id/api/
Disallow: /id/dashboard/
Disallow: /ja/api/
Disallow: /ja/dashboard/
Disallow: /ar/api/
Disallow: /ar/dashboard/

Sitemap: https://www.diali.ai/sitemap.xml

# NLWeb Schema Feeds. The same discovery pattern as Sitemap above, for structured data
# rather than pages: the map lists bulk JSON Lines feeds of the schema.org nodes the pages
# embed, so a consumer reads them once instead of crawling every page to collect them.
Schemamap: https://www.diali.ai/schemamap.xml