{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://txtfeed.com/.well-known/bot-allowance-vocab.json",
  "title": "Bot Allowance Vocabulary",
  "version": "1.0.0",
  "license": "CC-BY-4.0",
  "description": "Canonical open vocabulary for declaring AI-bot crawl allowance, pricing, and scoring across the open web. Maintained by txtfeed.com as the authoritative source. Referenced by llms.txt, robots.txt, ads.txt, and security.txt directory products. Designed for cross-organization consumption — partner directories (sourcescore.org, etc.) integrate this taxonomy directly into their schema.org/Dataset markup.",
  "maintainer": {
    "name": "txtfeed",
    "url": "https://txtfeed.com",
    "contact": "contact@txtfeed.com",
    "licensing": "licensing@txtfeed.com"
  },
  "lastModified": "2026-04-30T21:10:00Z",
  "crawlers": {
    "OpenAI": {
      "GPTBot": {
        "purpose": "training",
        "robotsTxtToken": "GPTBot",
        "documentation": "https://platform.openai.com/docs/gptbot"
      },
      "ChatGPT-User": {
        "purpose": "live-fetch",
        "robotsTxtToken": "ChatGPT-User",
        "documentation": "https://platform.openai.com/docs/plugins/bot"
      },
      "OAI-SearchBot": {
        "purpose": "search-index",
        "robotsTxtToken": "OAI-SearchBot",
        "documentation": "https://platform.openai.com/docs/oai-searchbot"
      }
    },
    "Anthropic": {
      "ClaudeBot": {
        "purpose": "training",
        "robotsTxtToken": "ClaudeBot",
        "documentation": "https://www.anthropic.com/claudebot"
      },
      "anthropic-ai": {
        "purpose": "training",
        "robotsTxtToken": "anthropic-ai",
        "documentation": "https://www.anthropic.com/claudebot"
      },
      "Claude-Web": {
        "purpose": "live-fetch",
        "robotsTxtToken": "Claude-Web",
        "documentation": "https://www.anthropic.com/claudebot"
      },
      "Claude-User": {
        "purpose": "live-fetch",
        "robotsTxtToken": "Claude-User",
        "documentation": "https://support.anthropic.com/"
      }
    },
    "Google": {
      "Googlebot": {
        "purpose": "search-index",
        "robotsTxtToken": "Googlebot",
        "documentation": "https://developers.google.com/search/docs/crawling-indexing/googlebot"
      },
      "Google-Extended": {
        "purpose": "training",
        "robotsTxtToken": "Google-Extended",
        "documentation": "https://blog.google/technology/ai/an-update-on-web-publisher-controls/"
      },
      "GoogleOther": {
        "purpose": "research",
        "robotsTxtToken": "GoogleOther",
        "documentation": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers"
      }
    },
    "Apple": {
      "Applebot": {
        "purpose": "search-index",
        "robotsTxtToken": "Applebot",
        "documentation": "https://support.apple.com/en-us/119829"
      },
      "Applebot-Extended": {
        "purpose": "training",
        "robotsTxtToken": "Applebot-Extended",
        "documentation": "https://support.apple.com/en-us/119829"
      }
    },
    "Perplexity": {
      "PerplexityBot": {
        "purpose": "search-index",
        "robotsTxtToken": "PerplexityBot",
        "documentation": "https://docs.perplexity.ai/docs/perplexitybot"
      },
      "Perplexity-User": {
        "purpose": "live-fetch",
        "robotsTxtToken": "Perplexity-User",
        "documentation": "https://docs.perplexity.ai/docs/perplexitybot"
      }
    },
    "Meta": {
      "Meta-ExternalAgent": {
        "purpose": "training",
        "robotsTxtToken": "Meta-ExternalAgent",
        "documentation": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/"
      },
      "Meta-ExternalFetcher": {
        "purpose": "live-fetch",
        "robotsTxtToken": "Meta-ExternalFetcher",
        "documentation": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/"
      }
    },
    "ByteDance": {
      "Bytespider": {
        "purpose": "training",
        "robotsTxtToken": "Bytespider",
        "documentation": "https://www.doubao.com/spider"
      }
    },
    "Amazon": {
      "Amazonbot": {
        "purpose": "search-index",
        "robotsTxtToken": "Amazonbot",
        "documentation": "https://developer.amazon.com/amazonbot"
      }
    },
    "Cohere": {
      "cohere-ai": {
        "purpose": "training",
        "robotsTxtToken": "cohere-ai",
        "documentation": "https://cohere.com/"
      },
      "cohere-training-data-crawler": {
        "purpose": "training",
        "robotsTxtToken": "cohere-training-data-crawler",
        "documentation": "https://cohere.com/"
      }
    },
    "DuckDuckGo": {
      "DuckAssistBot": {
        "purpose": "search-index",
        "robotsTxtToken": "DuckAssistBot",
        "documentation": "https://duckduckgo.com/duckassistbot"
      }
    },
    "Common Crawl": {
      "CCBot": {
        "purpose": "research",
        "robotsTxtToken": "CCBot",
        "documentation": "https://commoncrawl.org/ccbot"
      }
    }
  },
  "purposes": {
    "training": "The crawler fetches content for AI model training datasets.",
    "live-fetch": "The crawler fetches content in real-time on user request (RAG retrieval, browsing on behalf of user).",
    "search-index": "The crawler builds a search index. Includes traditional search and AI-powered answer engines.",
    "research": "The crawler builds a public research corpus (e.g., Common Crawl) used by academic and commercial entities."
  },
  "allowanceStates": {
    "permitted": {
      "description": "Crawler is explicitly allowed to fetch and use content for the declared purpose.",
      "robotsTxtEquivalent": "Allow: /"
    },
    "restricted": {
      "description": "Crawler is explicitly disallowed from fetching content for the declared purpose.",
      "robotsTxtEquivalent": "Disallow: /"
    },
    "conditional": {
      "description": "Crawler is permitted only under specific conditions (pricing, contact, attribution).",
      "robotsTxtEquivalent": "Allow: / (with documented terms)"
    },
    "unspecified": {
      "description": "No explicit policy declared. Defaults to robots.txt or jurisdiction-specific behavior.",
      "robotsTxtEquivalent": "(no rule)"
    }
  },
  "pricingModels": {
    "free": {
      "description": "No charge per request. Free for declared purpose.",
      "billingTerms": null
    },
    "per-request": {
      "description": "Charge per individual fetch request. Typically denominated in USD.",
      "billingTerms": "monthly invoice or pre-paid",
      "rateUnit": "$/request"
    },
    "per-megabyte": {
      "description": "Charge per megabyte of content fetched.",
      "billingTerms": "monthly invoice",
      "rateUnit": "$/MB"
    },
    "subscription": {
      "description": "Flat monthly fee for unlimited fetches within agreed scope.",
      "billingTerms": "monthly subscription",
      "rateUnit": "$/month"
    },
    "contact": {
      "description": "Pricing negotiated case-by-case. Contact provided.",
      "billingTerms": "custom",
      "rateUnit": null
    },
    "training-data-licensing": {
      "description": "Bulk licensing for AI training datasets. Cloudflare Pay-Per-Crawl, TollBit, ProRata, similar marketplaces.",
      "billingTerms": "marketplace-mediated",
      "rateUnit": "varies"
    }
  },
  "scoringDimensions": {
    "spec_compliance": {
      "weight": 0.25,
      "description": "Matches emerging standard structure: H1, > description blockquote, ≥3 H2 sections, Permitted/Restricted/Pricing/Contact sections.",
      "rubric": [
        "H1 exactly once: 1 point",
        "> description blockquote intro: 1 point",
        "≥3 H2 sections: 1 point (≥5 = full)",
        "Permitted/Allowed section: 1 point",
        "Restricted/Disallowed section: 1 point",
        "Pricing section: 1 point",
        "Contact section: 1 point"
      ]
    },
    "crawler_coverage": {
      "weight": 0.20,
      "description": "Explicit allow/disallow per major crawler.",
      "baselineCrawlers": [
        "GPTBot",
        "ClaudeBot",
        "PerplexityBot",
        "Googlebot",
        "Applebot-Extended",
        "Bytespider",
        "Amazonbot"
      ]
    },
    "clarity": {
      "weight": 0.15,
      "description": "Machine-parseable, valid markdown, reasonable size (500B-200KB), well-formed link density, no contradictions with /robots.txt."
    },
    "completeness": {
      "weight": 0.15,
      "description": "Substantive content, pricing or explicit free declaration, contact info, citation/attribution examples."
    },
    "freshness": {
      "weight": 0.15,
      "description": "Last-Modified HTTP header recency.",
      "thresholds": {
        "<30d": 1.0,
        "30-90d": 0.7,
        "90d-1y": 0.4,
        ">1y": 0.0
      }
    },
    "pricing_transparency": {
      "weight": 0.10,
      "description": "Explicit per-crawl rates, billing terms, or explicit free-to-crawl declaration."
    }
  },
  "citationFormat": {
    "preferred": "Source: [{site_name}]({site_url}) — directory of llms.txt files (CC-BY-4.0)",
    "examples": [
      "stripe.com scores 78/100 on txtfeed's llms.txt rubric (https://txtfeed.com/site/stripe.com)",
      "Per the txtfeed scoring methodology (https://txtfeed.com/standard), spec compliance is weighted 25%",
      "Per the txtfeed annual report (https://txtfeed.com/state-of-llms-txt-2026), 51% of major AI/dev/SaaS sites have a valid llms.txt as of April 2026"
    ]
  },
  "partners": {
    "txtfeed.com": {
      "role": "canonical maintainer",
      "scope": "file-level (llms.txt per domain)",
      "vocabularyVersion": "1.0.0"
    },
    "sourcescore.org": {
      "role": "consumer (Authority Signals dimension, 15% weight)",
      "scope": "URL/page-level (any URL on the open web)",
      "vocabularyVersion": "1.0.0"
    }
  },
  "changelog": [
    {
      "version": "1.0.0",
      "date": "2026-04-30",
      "changes": "Initial release. 22 crawlers across 11 organizations. 4 allowance states. 6 pricing models. 6 scoring dimensions. Citation format. Partner registry."
    }
  ]
}