{"openapi":"3.1.0","info":{"title":"txtfeed llms.txt Directory API","summary":"Open directory of llms.txt files across the web with transparent 6-dimension scoring.","description":"txtfeed is the canonical directory of llms.txt files across the web. Every domain in our corpus gets a transparent 0-100 score across 6 dimensions: Spec Compliance, Crawler Coverage, Clarity, Completeness, Freshness, and Pricing Transparency. This API exposes the directory programmatically — AI companies query it for real-time crawler-permission checks, bot-management providers license the dataset, and publishers monitor their own llms.txt + competitors. In the spirit of Have I Been Pwned: solo-operator-built, free public DB. Pricing tiers (Pro / Studio / API / Enterprise) are planned but not yet live; the free /v1/check endpoint will remain available.","version":"1.0.0","termsOfService":"https://txtfeed.com/terms","contact":{"name":"txtfeed","url":"https://txtfeed.com","email":"contact@txtfeed.com"},"license":{"name":"CC-BY-4.0","url":"https://creativecommons.org/licenses/by/4.0/","identifier":"CC-BY-4.0"},"x-logo":{"url":"https://txtfeed.com/icon-512.png","altText":"txtfeed"}},"servers":[{"url":"https://txtfeed.com","description":"Production"}],"externalDocs":{"description":"Scoring methodology — transparent 6-dimension rubric used to compute the 0-100 score","url":"https://txtfeed.com/standard"},"tags":[{"name":"directory","description":"Read-only directory lookup endpoints. Free for all permitted AI crawlers and human integrators (60 req/min courtesy limit; Pro tier raises to 600 req/min when launched)."}],"paths":{"/api/llms/v1/check":{"get":{"operationId":"checkDomain","summary":"Look up a domain's llms.txt score and metadata","description":"Returns the cached score + structural metadata for a domain in the txtfeed corpus. The corpus seeded on 2026-04-30 with 56 domains and is expanding daily. Unknown domains receive a `found: false` response with a polite request-inclusion link rather than a 404 — the integrator can decide whether to proceed (e.g., assume default crawl-permitted) or queue the domain for inclusion. Per-crawler allow/disallow resolution, per-crawl pricing, and real-time fetching of arbitrary uncached domains are planned but not yet supported (flagged in the `caveats` block of the success response).","tags":["directory"],"parameters":[{"name":"url","in":"query","required":true,"description":"Domain or URL to look up. Accepts bare domain (`stripe.com`), URL with scheme (`https://stripe.com/`), or path-prefixed URL (`https://stripe.com/docs`). The path component is stripped — txtfeed scores at the domain level. Mutually equivalent with the `domain` parameter (one is required).","schema":{"type":"string","example":"stripe.com"}},{"name":"domain","in":"query","required":false,"description":"Alias for `url`. Bare domain form. Mutually equivalent — provide one or the other.","schema":{"type":"string","example":"stripe.com"}},{"name":"crawler","in":"query","required":false,"description":"Reserved for future per-crawler resolution. Currently ignored; the response includes a `caveats.crawler_resolution_supported: false` field. Planned values include `GPTBot`, `ClaudeBot`, `PerplexityBot`, `Googlebot-Extended`, `Applebot-Extended`, `Bytespider`, `Amazonbot`, `Meta-ExternalAgent`, `CCBot`, `cohere-ai`, `DuckAssistBot`.","schema":{"type":"string","example":"GPTBot"}}],"responses":{"200":{"description":"Lookup completed. The response body always returns 200 — `found: true` for cached domains, `found: false` for uncached domains with a request-inclusion path. Integrators should branch on the `found` field rather than HTTP status.","content":{"application/json":{"schema":{"oneOf":[{"$ref":"#/components/schemas/CheckHit"},{"$ref":"#/components/schemas/CheckMiss"}]},"examples":{"cached_domain":{"summary":"Cached domain (stripe.com)","value":{"found":true,"domain":"stripe.com","url":"stripe.com","llms_txt_url":"https://stripe.com/llms.txt","allowed":null,"pricing":null,"contact":null,"last_modified":null,"score":57.4,"grade":"C+","category":"saas","rank_in_category":1,"structural":{"bytes":64229,"h1_count":1,"h2_count":28,"link_count":272,"has_quote_intro":true,"crawlers_mentioned":[]},"score_breakdown":{"spec_compliance":0.857,"crawler_coverage":0,"clarity":1,"completeness":1,"freshness":0,"pricing_transparency":0.6},"site_page":"https://txtfeed.com/site/stripe.com","methodology":"https://txtfeed.com/standard","fetched_at":"2026-04-30T18:49:57+00:00","api_version":"v1","caveats":{"crawler_resolution_supported":false,"pricing_resolution_supported":false,"realtime_fetch_supported":false}}},"uncached_domain":{"summary":"Uncached domain (request-inclusion)","value":{"found":false,"domain":"this-does-not-exist-12345.com","url":"this-does-not-exist-12345.com","reason":"Domain not in current txtfeed corpus. We seed-crawled 56 domains on 2026-04-30 and are expanding daily.","request_inclusion":"Email contact@txtfeed.com with the domain you want included; we fetch + score within 48 hours.","api_version":"v1"}}}}}},"400":{"description":"Bad request — missing or invalid `url` / `domain` parameter.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"},"examples":{"missing_param":{"value":{"error":"Missing required query parameter: url"}}}}}}}}}},"components":{"schemas":{"ScoreBreakdown":{"type":"object","description":"Per-dimension scores (0.0 — 1.0). Sum × 100 / 6 ≈ overall score, weighted equally across the six dimensions of the public methodology at /standard.","required":["spec_compliance","crawler_coverage","clarity","completeness","freshness","pricing_transparency"],"properties":{"spec_compliance":{"type":"number","format":"float","minimum":0,"maximum":1,"description":"Adherence to the published llms.txt specification (header presence, blockquote-intro, section structure)."},"crawler_coverage":{"type":"number","format":"float","minimum":0,"maximum":1,"description":"Explicit per-crawler directives. 1.0 when GPTBot, ClaudeBot, PerplexityBot, Googlebot-Extended, Applebot-Extended are all named."},"clarity":{"type":"number","format":"float","minimum":0,"maximum":1,"description":"Readability and structure of the file content."},"completeness":{"type":"number","format":"float","minimum":0,"maximum":1,"description":"Whether the file covers permitted paths, restricted paths, and policy beyond the bare minimum."},"freshness":{"type":"number","format":"float","minimum":0,"maximum":1,"description":"Inferred from Last-Modified header + visible dates. Stale files (>180 days) score lower."},"pricing_transparency":{"type":"number","format":"float","minimum":0,"maximum":1,"description":"Whether crawl pricing (e.g., Cloudflare Pay-Per-Crawl integration) is declared."}}},"Structural":{"type":"object","description":"Raw structural metadata extracted from the fetched llms.txt file.","required":["bytes","h1_count","h2_count","link_count","has_quote_intro","crawlers_mentioned"],"properties":{"bytes":{"type":"integer","description":"Size of the llms.txt response body in bytes."},"h1_count":{"type":"integer"},"h2_count":{"type":"integer"},"link_count":{"type":"integer","description":"Count of unique outbound links in the file."},"has_quote_intro":{"type":"boolean","description":"Whether the file uses the recommended `> ...` blockquote summary immediately after the H1."},"crawlers_mentioned":{"type":"array","items":{"type":"string"},"description":"User-agent strings explicitly named in the file. Currently mostly empty across the corpus."}}},"Caveats":{"type":"object","description":"Honest declaration of what this API currently does NOT compute. Integrators should branch on these flags rather than assume.","required":["crawler_resolution_supported","pricing_resolution_supported","realtime_fetch_supported"],"properties":{"crawler_resolution_supported":{"type":"boolean","description":"False today. When true, the `allowed` field will return a per-crawler boolean."},"pricing_resolution_supported":{"type":"boolean","description":"False today. When true, the `pricing` field will return per-crawler USD pricing if declared."},"realtime_fetch_supported":{"type":"boolean","description":"False today. When true, uncached domains will be fetched + scored on demand (rate-limited)."}}},"CheckHit":{"type":"object","description":"Cached domain lookup hit. Returned when `found: true`.","required":["found","domain","url","llms_txt_url","score","grade","category","rank_in_category","structural","score_breakdown","site_page","methodology","fetched_at","api_version","caveats"],"properties":{"found":{"type":"boolean","enum":[true]},"domain":{"type":"string","example":"stripe.com"},"url":{"type":"string","example":"stripe.com"},"llms_txt_url":{"type":"string","format":"uri","example":"https://stripe.com/llms.txt"},"allowed":{"type":"null","description":"Reserved for future per-crawler resolution (see caveats.crawler_resolution_supported)."},"pricing":{"type":"null","description":"Reserved for future per-crawler pricing (see caveats.pricing_resolution_supported)."},"contact":{"type":["string","null"],"description":"Contact email or URL declared in the file, if any. Currently null across the corpus."},"last_modified":{"type":["string","null"],"format":"date-time","description":"HTTP Last-Modified header if returned by origin."},"score":{"type":"number","format":"float","minimum":0,"maximum":100,"description":"Overall 0-100 score. Weighted equally across the 6 dimensions of score_breakdown."},"grade":{"type":"string","enum":["A","B+","B","C+","C","D","F"],"description":"Human-readable letter grade derived from `score`. ≥80=A, ≥70=B+, ≥60=B, ≥50=C+, ≥40=C, ≥25=D, else F."},"category":{"type":"string","description":"Categorical bucket of the domain — saas, dev-tools, docs, media, marketplace, etc.","example":"saas"},"rank_in_category":{"type":"integer","description":"1-indexed rank within the category by score."},"structural":{"$ref":"#/components/schemas/Structural"},"score_breakdown":{"$ref":"#/components/schemas/ScoreBreakdown"},"site_page":{"type":"string","format":"uri","description":"Human-readable directory page for this domain.","example":"https://txtfeed.com/site/stripe.com"},"methodology":{"type":"string","format":"uri","example":"https://txtfeed.com/standard"},"fetched_at":{"type":"string","format":"date-time","description":"When txtfeed last fetched + scored this domain."},"api_version":{"type":"string","enum":["v1"]},"caveats":{"$ref":"#/components/schemas/Caveats"}}},"CheckMiss":{"type":"object","description":"Uncached domain. Returned when `found: false`. Integrators should branch on the `found` field and either fall back to default-crawl-permitted assumption, queue the domain via `request_inclusion`, or both.","required":["found","url","reason","request_inclusion","api_version"],"properties":{"found":{"type":"boolean","enum":[false]},"domain":{"type":["string","null"]},"url":{"type":"string"},"reason":{"type":"string","example":"Domain not in current txtfeed corpus. We seed-crawled 56 domains on 2026-04-30 and are expanding daily."},"request_inclusion":{"type":"string","example":"Email contact@txtfeed.com with the domain you want included; we fetch + score within 48 hours."},"api_version":{"type":"string","enum":["v1"]}}},"Error":{"type":"object","required":["error"],"properties":{"error":{"type":"string"}}}}}}