{
 "question": "What share of the top 10,000 websites publish a valid llms.txt (and llms-full.txt), and what do those files contain?",
 "population": "Tranco top-10,000 domains with a live homepage (HTTP status below 400, more than 200 bytes) when fetched (n=5,902).",
 "inclusion": "llms.txt counted as valid when /llms.txt returns HTTP 200, is not HTML, is not redirected to another path, is at least 20 characters and opens with a Markdown H1 as the llmstxt.org proposal specifies.",
 "exclusion": "Soft 404s (HTML at the path), redirects elsewhere, empty files; these are reported separately.",
 "calculations": [
  "Adoption = valid files / live homepages.",
  "Content features (size, links, H2 sections, blockquote summary) measured on the valid files.",
  "95% intervals: Wilson score for shares; Newcombe hybrid score for the top-1,000 minus 5,001-10,000 difference; Cochran-Armitage test for a linear trend over 10 rank bands of 1,000.",
  "Control path (28 Sep): /llms-7f3a9c21b8e4d056.txt, a name no site publishes, fetched once per live domain with the same collector. A file is a catch-all response when the control path returns the same body.",
  "Test-retest (28 Sep): /llms.txt fetched again for every live domain and classified with the same rule; agreement and Cohen's kappa between the two dates.",
  "Conformance ladder on valid files: Markdown H1; plus a blockquote summary; plus H2 sections containing Markdown list links; plus at least one link to a .md page."
 ],
 "limitations": [
  "Domains that blocked our user agent at the homepage are outside the denominator.",
  "Presence of a file says nothing about whether any AI system reads it.",
  "Two fetches, two days apart; a site's state can change between quarterly editions.",
  "One user agent; a site may answer AI crawlers' user agents differently."
 ],
 "update": "quarterly",
 "@context": "https://schema.org",
 "@type": "Dataset",
 "name": "How many websites have an llms.txt file? 2026 adoption data",
 "version": "1.1",
 "dateCreated": "2026-09-26",
 "creator": {
  "@type": "Organization",
  "name": "Underneath",
  "url": "https://underneath.agency"
 },
 "url": "https://underneath.agency/research/llms-txt-adoption-study",
 "temporalCoverage": "2026-09-26/2026-09-28",
 "isAccessibleForFree": true,
 "license": "https://creativecommons.org/licenses/by/4.0/",
 "code": "cite/pipeline/ (collection, validation and analysis scripts)",
 "variableMeasured": [
  "sample",
  "s2_llms_state",
  "s2_llms_full_state",
  "s2_valid_n",
  "s2_valid_pct",
  "s2_any_text_n",
  "s2_any_text_pct",
  "s2_by_tier",
  "s2_valid_with_full",
  "s2_features",
  "s2_llms_but_blocks_ai",
  "s2_valid_top_examples",
  "s2_derived"
 ],
 "stage_measured": "Publication only: whether a site serves a file at /llms.txt that meets the proposal's format. The study does not measure whether any AI system requests the file (discovery), uses it to find pages (retrieval), cites the site, or uses its content in answers.",
 "dateModified": "2026-09-28"
}