{
 "study": "s1",
 "version": "1.2",
 "collected": "2026-09-26",
 "statistics": {
  "sample": {
   "list": "Tranco top 1M, list ID L5PZ4 (downloaded 2026-09-26)",
   "domains_attempted": 10000,
   "domains_with_live_homepage": 5902,
   "crawl_window": [
    "2026-09-26T04:16:26.234430Z",
    "2026-09-26T04:38:00Z"
   ]
  },
  "s1_robots_state_all_attempted": {
   "ok": 5572,
   "missing": 1431,
   "unreachable": 2467,
   "invalid": 485,
   "server_error": 45
  },
  "s1_robots_state_live_sites": {
   "ok": 4959,
   "invalid": 448,
   "missing": 413,
   "unreachable": 77,
   "server_error": 5
  },
  "s1_n_parsed": 5572,
  "s1_per_bot": {
   "GPTBot": {
    "operator": "OpenAI",
    "purpose": "training",
    "blocked_root": 849,
    "pct_blocked": 15.2,
    "named_in_file": 1004,
    "pct_named": 18.0,
    "blocked_by_name": 676,
    "pct_blocked_by_name": 12.1,
    "partial": 3598,
    "pct_blocked_by_tier": {
     "1-1,000": 20.7,
     "1,001-5,000": 15.4,
     "5,001-10,000": 14.2
    }
   },
   "OAI-SearchBot": {
    "operator": "OpenAI",
    "purpose": "search",
    "blocked_root": 415,
    "pct_blocked": 7.4,
    "named_in_file": 561,
    "pct_named": 10.1,
    "blocked_by_name": 242,
    "pct_blocked_by_name": 4.3,
    "partial": 3987,
    "pct_blocked_by_tier": {
     "1-1,000": 11.0,
     "1,001-5,000": 7.4,
     "5,001-10,000": 6.8
    }
   },
   "ChatGPT-User": {
    "operator": "OpenAI",
    "purpose": "user",
    "blocked_root": 539,
    "pct_blocked": 9.7,
    "named_in_file": 683,
    "pct_named": 12.3,
    "blocked_by_name": 365,
    "pct_blocked_by_name": 6.6,
    "partial": 3885,
    "pct_blocked_by_tier": {
     "1-1,000": 13.4,
     "1,001-5,000": 10.0,
     "5,001-10,000": 8.8
    }
   },
   "ClaudeBot": {
    "operator": "Anthropic",
    "purpose": "training",
    "blocked_root": 788,
    "pct_blocked": 14.1,
    "named_in_file": 852,
    "pct_named": 15.3,
    "blocked_by_name": 601,
    "pct_blocked_by_name": 10.8,
    "partial": 3668,
    "pct_blocked_by_tier": {
     "1-1,000": 20.9,
     "1,001-5,000": 14.3,
     "5,001-10,000": 12.8
    }
   },
   "Claude-SearchBot": {
    "operator": "Anthropic",
    "purpose": "search",
    "blocked_root": 398,
    "pct_blocked": 7.1,
    "named_in_file": 343,
    "pct_named": 6.2,
    "blocked_by_name": 208,
    "pct_blocked_by_name": 3.7,
    "partial": 4068,
    "pct_blocked_by_tier": {
     "1-1,000": 12.4,
     "1,001-5,000": 7.1,
     "5,001-10,000": 6.2
    }
   },
   "Claude-User": {
    "operator": "Anthropic",
    "purpose": "user",
    "blocked_root": 410,
    "pct_blocked": 7.4,
    "named_in_file": 350,
    "pct_named": 6.3,
    "blocked_by_name": 219,
    "pct_blocked_by_name": 3.9,
    "partial": 4059,
    "pct_blocked_by_tier": {
     "1-1,000": 13.2,
     "1,001-5,000": 7.4,
     "5,001-10,000": 6.3
    }
   },
   "anthropic-ai": {
    "operator": "Anthropic",
    "purpose": "legacy",
    "blocked_root": 660,
    "pct_blocked": 11.8,
    "named_in_file": 573,
    "pct_named": 10.3,
    "blocked_by_name": 461,
    "pct_blocked_by_name": 8.3,
    "partial": 3844,
    "pct_blocked_by_tier": {
     "1-1,000": 15.8,
     "1,001-5,000": 12.3,
     "5,001-10,000": 10.8
    }
   },
   "Claude-Web": {
    "operator": "Anthropic",
    "purpose": "legacy",
    "blocked_root": 579,
    "pct_blocked": 10.4,
    "named_in_file": 467,
    "pct_named": 8.4,
    "blocked_by_name": 379,
    "pct_blocked_by_name": 6.8,
    "partial": 3919,
    "pct_blocked_by_tier": {
     "1-1,000": 16.0,
     "1,001-5,000": 10.7,
     "5,001-10,000": 9.1
    }
   },
   "PerplexityBot": {
    "operator": "Perplexity",
    "purpose": "search",
    "blocked_root": 607,
    "pct_blocked": 10.9,
    "named_in_file": 723,
    "pct_named": 13.0,
    "blocked_by_name": 429,
    "pct_blocked_by_name": 7.7,
    "partial": 3807,
    "pct_blocked_by_tier": {
     "1-1,000": 16.8,
     "1,001-5,000": 11.1,
     "5,001-10,000": 9.7
    }
   },
   "Perplexity-User": {
    "operator": "Perplexity",
    "purpose": "user",
    "blocked_root": 427,
    "pct_blocked": 7.7,
    "named_in_file": 350,
    "pct_named": 6.3,
    "blocked_by_name": 232,
    "pct_blocked_by_name": 4.2,
    "partial": 4046,
    "pct_blocked_by_tier": {
     "1-1,000": 14.0,
     "1,001-5,000": 7.6,
     "5,001-10,000": 6.6
    }
   },
   "Google-Extended": {
    "operator": "Google",
    "purpose": "training",
    "blocked_root": 686,
    "pct_blocked": 12.3,
    "named_in_file": 761,
    "pct_named": 13.7,
    "blocked_by_name": 506,
    "pct_blocked_by_name": 9.1,
    "partial": 3757,
    "pct_blocked_by_tier": {
     "1-1,000": 18.3,
     "1,001-5,000": 12.5,
     "5,001-10,000": 11.1
    }
   },
   "Applebot-Extended": {
    "operator": "Apple",
    "purpose": "training",
    "blocked_root": 649,
    "pct_blocked": 11.6,
    "named_in_file": 569,
    "pct_named": 10.2,
    "blocked_by_name": 457,
    "pct_blocked_by_name": 8.2,
    "partial": 3854,
    "pct_blocked_by_tier": {
     "1-1,000": 17.2,
     "1,001-5,000": 12.0,
     "5,001-10,000": 10.4
    }
   },
   "CCBot": {
    "operator": "Common Crawl",
    "purpose": "training",
    "blocked_root": 904,
    "pct_blocked": 16.2,
    "named_in_file": 848,
    "pct_named": 15.2,
    "blocked_by_name": 710,
    "pct_blocked_by_name": 12.7,
    "partial": 3610,
    "pct_blocked_by_tier": {
     "1-1,000": 24.3,
     "1,001-5,000": 16.0,
     "5,001-10,000": 15.0
    }
   },
   "Bytespider": {
    "operator": "ByteDance",
    "purpose": "training",
    "blocked_root": 855,
    "pct_blocked": 15.3,
    "named_in_file": 737,
    "pct_named": 13.2,
    "blocked_by_name": 657,
    "pct_blocked_by_name": 11.8,
    "partial": 3670,
    "pct_blocked_by_tier": {
     "1-1,000": 23.9,
     "1,001-5,000": 15.0,
     "5,001-10,000": 14.1
    }
   },
   "Meta-ExternalAgent": {
    "operator": "Meta",
    "purpose": "training",
    "blocked_root": 688,
    "pct_blocked": 12.3,
    "named_in_file": 596,
    "pct_named": 10.7,
    "blocked_by_name": 494,
    "pct_blocked_by_name": 8.9,
    "partial": 3818,
    "pct_blocked_by_tier": {
     "1-1,000": 19.5,
     "1,001-5,000": 13.1,
     "5,001-10,000": 10.5
    }
   },
   "meta-externalfetcher": {
    "operator": "Meta",
    "purpose": "user",
    "blocked_root": 463,
    "pct_blocked": 8.3,
    "named_in_file": 305,
    "pct_named": 5.5,
    "blocked_by_name": 266,
    "pct_blocked_by_name": 4.8,
    "partial": 4042,
    "pct_blocked_by_tier": {
     "1-1,000": 12.6,
     "1,001-5,000": 8.7,
     "5,001-10,000": 7.2
    }
   },
   "Amazonbot": {
    "operator": "Amazon",
    "purpose": "training",
    "blocked_root": 649,
    "pct_blocked": 11.6,
    "named_in_file": 573,
    "pct_named": 10.3,
    "blocked_by_name": 458,
    "pct_blocked_by_name": 8.2,
    "partial": 3851,
    "pct_blocked_by_tier": {
     "1-1,000": 15.8,
     "1,001-5,000": 11.5,
     "5,001-10,000": 11.0
    }
   },
   "cohere-ai": {
    "operator": "Cohere",
    "purpose": "legacy",
    "blocked_root": 653,
    "pct_blocked": 11.7,
    "named_in_file": 522,
    "pct_named": 9.4,
    "blocked_by_name": 453,
    "pct_blocked_by_name": 8.1,
    "partial": 3864,
    "pct_blocked_by_tier": {
     "1-1,000": 17.6,
     "1,001-5,000": 12.0,
     "5,001-10,000": 10.5
    }
   },
   "DuckAssistBot": {
    "operator": "DuckDuckGo",
    "purpose": "search",
    "blocked_root": 433,
    "pct_blocked": 7.8,
    "named_in_file": 297,
    "pct_named": 5.3,
    "blocked_by_name": 236,
    "pct_blocked_by_name": 4.2,
    "partial": 4057,
    "pct_blocked_by_tier": {
     "1-1,000": 12.6,
     "1,001-5,000": 7.8,
     "5,001-10,000": 6.9
    }
   },
   "MistralAI-User": {
    "operator": "Mistral",
    "purpose": "user",
    "blocked_root": 397,
    "pct_blocked": 7.1,
    "named_in_file": 234,
    "pct_named": 4.2,
    "blocked_by_name": 192,
    "pct_blocked_by_name": 3.4,
    "partial": 4098,
    "pct_blocked_by_tier": {
     "1-1,000": 13.0,
     "1,001-5,000": 7.4,
     "5,001-10,000": 5.9
    }
   },
   "Googlebot": {
    "operator": "Google",
    "purpose": "control",
    "blocked_root": 128,
    "pct_blocked": 2.3,
    "named_in_file": 514,
    "pct_named": 9.2,
    "blocked_by_name": 8,
    "pct_blocked_by_name": 0.1,
    "partial": 4301,
    "pct_blocked_by_tier": {
     "1-1,000": 2.2,
     "1,001-5,000": 2.6,
     "5,001-10,000": 2.1
    }
   },
   "Bingbot": {
    "operator": "Microsoft",
    "purpose": "control",
    "blocked_root": 173,
    "pct_blocked": 3.1,
    "named_in_file": 415,
    "pct_named": 7.4,
    "blocked_by_name": 35,
    "pct_blocked_by_name": 0.6,
    "partial": 4230,
    "pct_blocked_by_tier": {
     "1-1,000": 2.8,
     "1,001-5,000": 3.2,
     "5,001-10,000": 3.1
    }
   },
   "Applebot": {
    "operator": "Apple",
    "purpose": "control",
    "blocked_root": 245,
    "pct_blocked": 4.4,
    "named_in_file": 238,
    "pct_named": 4.3,
    "blocked_by_name": 76,
    "pct_blocked_by_name": 1.4,
    "partial": 4211,
    "pct_blocked_by_tier": {
     "1-1,000": 4.9,
     "1,001-5,000": 4.3,
     "5,001-10,000": 4.4
    }
   }
  },
  "s1_combos": {
   "gptbot_blocked": {
    "n": 849,
    "pct": 15.2
   },
   "gptbot_blocked_oai_search_allowed": {
    "n": 439,
    "pct": 7.9
   },
   "oai_search_blocked": {
    "n": 415,
    "pct": 7.4
   },
   "oai_search_blocked_googlebot_allowed": {
    "n": 291,
    "pct": 5.2
   },
   "all_three_openai_blocked": {
    "n": 395,
    "pct": 7.1
   },
   "claudebot_blocked_searchbot_allowed": {
    "n": 391,
    "pct": 7.0
   },
   "claude_search_blocked": {
    "n": 398,
    "pct": 7.1
   },
   "google_extended_blocked": {
    "n": 686,
    "pct": 12.3
   },
   "google_extended_blocked_googlebot_allowed": {
    "n": 562,
    "pct": 10.1
   },
   "any_ai_bot_blocked": {
    "n": 1154,
    "pct": 20.7
   },
   "all_search_ai_blocked": {
    "n": 318,
    "pct": 5.7
   },
   "wildcard_disallow_root": {
    "n": 214,
    "pct": 3.8
   }
  },
  "s1_any_ai_named_pct": 24.2,
  "s1_any_ai_blocked_by_tier": {
   "1-1,000": 30.0,
   "1,001-5,000": 20.5,
   "5,001-10,000": 19.2
  },
  "s1_tier_n": {
   "1-1,000": 507,
   "1,001-5,000": 2196,
   "5,001-10,000": 2869
  },
  "s1_content_signal": {
   "n": 159,
   "pct": 2.9,
   "values": [
    [
     "search=yes",
     318
    ],
    [
     "ai-input=yes",
     204
    ],
    [
     "ai-train=no",
     204
    ],
    [
     "ai-train=yes",
     104
    ],
    [
     "ai-input=no",
     30
    ],
    [
     "use=reference",
     22
    ],
    [
     "search=no",
     2
    ],
    [
     "/next/ai-train=no",
     1
    ],
    [
     "/about/whats-newai-train=yes",
     1
    ],
    [
     "/features/ai-train=yes",
     1
    ],
    [
     "/help/ai-train=yes",
     1
    ],
    [
     "/integrations/ai-train=yes",
     1
    ]
   ]
  },
  "s1_robots_bytes_median": 602.0,
  "s1_derived": {
   "pct_of_gptbot_blockers_allowing_oai_searchbot": 51.7,
   "pct_of_claudebot_blockers_allowing_claude_searchbot": 49.6,
   "gptbot_vs_googlebot_block_ratio": 6.6,
   "top1000_gptbot_pct": 20.7,
   "robots_missing_pct_of_live": 7.0
  },
  "s1_content_signal_sites": {
   "total": 159,
   "ai_train_no": 102,
   "ai_train_yes": 60,
   "search_yes": 158,
   "ai_input_yes": 117,
   "ai_input_no": 22,
   "ai_train_no_pct": 64.2,
   "ai_train_yes_pct": 37.7,
   "search_yes_pct": 99.4,
   "ai_input_yes_pct": 73.6,
   "ai_input_no_pct": 13.8
  },
  "s1_v11": {
   "reproduces_version_1_0": true,
   "headline": {
    "GPTBot": {
     "n": 849,
     "of": 5572,
     "pct": 15.2,
     "ci95": [
      14.3,
      16.2
     ]
    },
    "OAI-SearchBot": {
     "n": 415,
     "of": 5572,
     "pct": 7.4,
     "ci95": [
      6.8,
      8.2
     ]
    },
    "ClaudeBot": {
     "n": 788,
     "of": 5572,
     "pct": 14.1,
     "ci95": [
      13.3,
      15.1
     ]
    },
    "Claude-SearchBot": {
     "n": 398,
     "of": 5572,
     "pct": 7.1,
     "ci95": [
      6.5,
      7.8
     ]
    },
    "PerplexityBot": {
     "n": 607,
     "of": 5572,
     "pct": 10.9,
     "ci95": [
      10.1,
      11.7
     ]
    },
    "CCBot": {
     "n": 904,
     "of": 5572,
     "pct": 16.2,
     "ci95": [
      15.3,
      17.2
     ]
    },
    "Google-Extended": {
     "n": 686,
     "of": 5572,
     "pct": 12.3,
     "ci95": [
      11.5,
      13.2
     ]
    },
    "Googlebot": {
     "n": 128,
     "of": 5572,
     "pct": 2.3,
     "ci95": [
      1.9,
      2.7
     ]
    },
    "Bytespider": {
     "n": 855,
     "of": 5572,
     "pct": 15.3,
     "ci95": [
      14.4,
      16.3
     ]
    },
    "Claude-User": {
     "n": 410,
     "of": 5572,
     "pct": 7.4,
     "ci95": [
      6.7,
      8.1
     ]
    },
    "ChatGPT-User": {
     "n": 539,
     "of": 5572,
     "pct": 9.7,
     "ci95": [
      8.9,
      10.5
     ]
    },
    "any_ai": {
     "n": 1154,
     "of": 5572,
     "pct": 20.7,
     "ci95": [
      19.7,
      21.8
     ]
    },
    "wildcard_disallow_root": {
     "n": 214,
     "of": 5572,
     "pct": 3.8,
     "ci95": [
      3.4,
      4.4
     ]
    },
    "gptbot_blockers_allowing_oai_search": {
     "n": 439,
     "of": 849,
     "pct": 51.7,
     "ci95": [
      48.3,
      55.1
     ]
    },
    "claudebot_blockers_allowing_claude_search": {
     "n": 391,
     "of": 788,
     "pct": 49.6,
     "ci95": [
      46.1,
      53.1
     ]
    },
    "oai_search_blocked_googlebot_allowed": {
     "n": 291,
     "of": 5572,
     "pct": 5.2,
     "ci95": [
      4.7,
      5.8
     ]
    }
   },
   "gradient": {
    "all_readable": {
     "sample": "all readable files",
     "n": 5572,
     "any_ai": {
      "by_tier": {
       "1-1,000": {
        "n": 152,
        "of": 507,
        "pct": 30.0,
        "ci95": [
         26.2,
         34.1
        ]
       },
       "1,001-5,000": {
        "n": 451,
        "of": 2196,
        "pct": 20.5,
        "ci95": [
         18.9,
         22.3
        ]
       },
       "5,001-10,000": {
        "n": 551,
        "of": 2869,
        "pct": 19.2,
        "ci95": [
         17.8,
         20.7
        ]
       }
      },
      "top1000_minus_5001_10000": {
       "diff_pp": 10.8,
       "ci95_pp": [
        6.7,
        15.1
       ]
      },
      "trend_3_tiers": {
       "z": -4.57,
       "p": 4.9e-06
      },
      "trend_10_bands": {
       "z": -5.03,
       "p": 4.8e-07
      },
      "by_1000_band_pct": {
       "1-1000": 30.0,
       "1001-2000": 21.0,
       "2001-3000": 21.4,
       "3001-4000": 19.1,
       "4001-5000": 20.6,
       "5001-6000": 21.3,
       "6001-7000": 22.9,
       "7001-8000": 19.9,
       "8001-9000": 19.3,
       "9001-10000": 13.1
      }
     },
     "gptbot": {
      "by_tier": {
       "1-1,000": {
        "n": 105,
        "of": 507,
        "pct": 20.7,
        "ci95": [
         17.4,
         24.5
        ]
       },
       "1,001-5,000": {
        "n": 338,
        "of": 2196,
        "pct": 15.4,
        "ci95": [
         13.9,
         17.0
        ]
       },
       "5,001-10,000": {
        "n": 406,
        "of": 2869,
        "pct": 14.2,
        "ci95": [
         12.9,
         15.5
        ]
       }
      },
      "top1000_minus_5001_10000": {
       "diff_pp": 6.6,
       "ci95_pp": [
        3.0,
        10.5
       ]
      },
      "trend_3_tiers": {
       "z": -3.36,
       "p": 0.00077
      },
      "trend_10_bands": {
       "z": -3.89,
       "p": 0.0001
      },
      "by_1000_band_pct": {
       "1-1000": 20.7,
       "1001-2000": 15.3,
       "2001-3000": 16.3,
       "3001-4000": 15.7,
       "4001-5000": 14.3,
       "5001-6000": 15.9,
       "6001-7000": 16.7,
       "7001-8000": 13.9,
       "8001-9000": 14.4,
       "9001-10000": 10.1
      }
     },
     "oai_search": {
      "by_tier": {
       "1-1,000": {
        "n": 56,
        "of": 507,
        "pct": 11.0,
        "ci95": [
         8.6,
         14.1
        ]
       },
       "1,001-5,000": {
        "n": 163,
        "of": 2196,
        "pct": 7.4,
        "ci95": [
         6.4,
         8.6
        ]
       },
       "5,001-10,000": {
        "n": 196,
        "of": 2869,
        "pct": 6.8,
        "ci95": [
         6.0,
         7.8
        ]
       }
      },
      "top1000_minus_5001_10000": {
       "diff_pp": 4.2,
       "ci95_pp": [
        1.6,
        7.4
       ]
      },
      "trend_3_tiers": {
       "z": -2.81,
       "p": 0.005
      },
      "trend_10_bands": {
       "z": -2.98,
       "p": 0.0028
      },
      "by_1000_band_pct": {
       "1-1000": 11.0,
       "1001-2000": 7.2,
       "2001-3000": 8.9,
       "3001-4000": 6.6,
       "4001-5000": 7.1,
       "5001-6000": 7.7,
       "6001-7000": 8.4,
       "7001-8000": 5.8,
       "8001-9000": 7.3,
       "9001-10000": 5.1
      }
     }
    },
    "live_homepage": {
     "sample": "readable files of domains with a live homepage",
     "n": 4959,
     "any_ai": {
      "by_tier": {
       "1-1,000": {
        "n": 119,
        "of": 437,
        "pct": 27.2,
        "ci95": [
         23.3,
         31.6
        ]
       },
       "1,001-5,000": {
        "n": 360,
        "of": 1971,
        "pct": 18.3,
        "ci95": [
         16.6,
         20.0
        ]
       },
       "5,001-10,000": {
        "n": 467,
        "of": 2551,
        "pct": 18.3,
        "ci95": [
         16.9,
         19.9
        ]
       }
      },
      "top1000_minus_5001_10000": {
       "diff_pp": 8.9,
       "ci95_pp": [
        4.7,
        13.5
       ]
      },
      "trend_3_tiers": {
       "z": -3.08,
       "p": 0.0021
      },
      "trend_10_bands": {
       "z": -3.91,
       "p": 9.1e-05
      },
      "by_1000_band_pct": {
       "1-1000": 27.2,
       "1001-2000": 19.6,
       "2001-3000": 18.6,
       "3001-4000": 17.5,
       "4001-5000": 17.3,
       "5001-6000": 21.1,
       "6001-7000": 22.0,
       "7001-8000": 18.5,
       "8001-9000": 19.3,
       "9001-10000": 11.5
      }
     },
     "gptbot": {
      "by_tier": {
       "1-1,000": {
        "n": 80,
        "of": 437,
        "pct": 18.3,
        "ci95": [
         15.0,
         22.2
        ]
       },
       "1,001-5,000": {
        "n": 268,
        "of": 1971,
        "pct": 13.6,
        "ci95": [
         12.2,
         15.2
        ]
       },
       "5,001-10,000": {
        "n": 342,
        "of": 2551,
        "pct": 13.4,
        "ci95": [
         12.1,
         14.8
        ]
       }
      },
      "top1000_minus_5001_10000": {
       "diff_pp": 4.9,
       "ci95_pp": [
        1.3,
        9.0
       ]
      },
      "trend_3_tiers": {
       "z": -2.03,
       "p": 0.042
      },
      "trend_10_bands": {
       "z": -2.93,
       "p": 0.0034
      },
      "by_1000_band_pct": {
       "1-1000": 18.3,
       "1001-2000": 13.8,
       "2001-3000": 14.3,
       "3001-4000": 14.0,
       "4001-5000": 12.3,
       "5001-6000": 15.5,
       "6001-7000": 16.3,
       "7001-8000": 13.4,
       "8001-9000": 14.2,
       "9001-10000": 8.2
      }
     },
     "oai_search": {
      "by_tier": {
       "1-1,000": {
        "n": 44,
        "of": 437,
        "pct": 10.1,
        "ci95": [
         7.6,
         13.2
        ]
       },
       "1,001-5,000": {
        "n": 132,
        "of": 1971,
        "pct": 6.7,
        "ci95": [
         5.7,
         7.9
        ]
       },
       "5,001-10,000": {
        "n": 161,
        "of": 2551,
        "pct": 6.3,
        "ci95": [
         5.4,
         7.3
        ]
       }
      },
      "top1000_minus_5001_10000": {
       "diff_pp": 3.8,
       "ci95_pp": [
        1.1,
        7.0
       ]
      },
      "trend_3_tiers": {
       "z": -2.32,
       "p": 0.02
      },
      "trend_10_bands": {
       "z": -2.89,
       "p": 0.0038
      },
      "by_1000_band_pct": {
       "1-1000": 10.1,
       "1001-2000": 6.7,
       "2001-3000": 8.8,
       "3001-4000": 5.3,
       "4001-5000": 6.1,
       "5001-6000": 7.8,
       "6001-7000": 7.6,
       "7001-8000": 5.4,
       "8001-9000": 7.2,
       "9001-10000": 3.8
      }
     }
    },
    "no_wildcard_root_block": {
     "sample": "readable files that do not disallow the root for unnamed bots",
     "n": 5358,
     "any_ai": {
      "by_tier": {
       "1-1,000": {
        "n": 117,
        "of": 471,
        "pct": 24.8,
        "ci95": [
         21.2,
         28.9
        ]
       },
       "1,001-5,000": {
        "n": 364,
        "of": 2109,
        "pct": 17.3,
        "ci95": [
         15.7,
         18.9
        ]
       },
       "5,001-10,000": {
        "n": 460,
        "of": 2778,
        "pct": 16.6,
        "ci95": [
         15.2,
         18.0
        ]
       }
      },
      "top1000_minus_5001_10000": {
       "diff_pp": 8.3,
       "ci95_pp": [
        4.4,
        12.6
       ]
      },
      "trend_3_tiers": {
       "z": -3.44,
       "p": 0.00058
      },
      "trend_10_bands": {
       "z": -4.48,
       "p": 7.6e-06
      },
      "by_1000_band_pct": {
       "1-1000": 24.8,
       "1001-2000": 18.2,
       "2001-3000": 17.2,
       "3001-4000": 16.1,
       "4001-5000": 17.4,
       "5001-6000": 19.2,
       "6001-7000": 20.9,
       "7001-8000": 17.8,
       "8001-9000": 16.1,
       "9001-10000": 9.4
      }
     },
     "gptbot": {
      "by_tier": {
       "1-1,000": {
        "n": 76,
        "of": 471,
        "pct": 16.1,
        "ci95": [
         13.1,
         19.7
        ]
       },
       "1,001-5,000": {
        "n": 262,
        "of": 2109,
        "pct": 12.4,
        "ci95": [
         11.1,
         13.9
        ]
       },
       "5,001-10,000": {
        "n": 318,
        "of": 2778,
        "pct": 11.4,
        "ci95": [
         10.3,
         12.7
        ]
       }
      },
      "top1000_minus_5001_10000": {
       "diff_pp": 4.7,
       "ci95_pp": [
        1.4,
        8.4
       ]
      },
      "trend_3_tiers": {
       "z": -2.6,
       "p": 0.0094
      },
      "trend_10_bands": {
       "z": -3.76,
       "p": 0.00017
      },
      "by_1000_band_pct": {
       "1-1000": 16.1,
       "1001-2000": 12.6,
       "2001-3000": 12.5,
       "3001-4000": 13.4,
       "4001-5000": 11.2,
       "5001-6000": 13.8,
       "6001-7000": 14.6,
       "7001-8000": 12.0,
       "8001-9000": 11.0,
       "9001-10000": 6.2
      }
     },
     "oai_search": {
      "by_tier": {
       "1-1,000": {
        "n": 30,
        "of": 471,
        "pct": 6.4,
        "ci95": [
         4.5,
         8.9
        ]
       },
       "1,001-5,000": {
        "n": 91,
        "of": 2109,
        "pct": 4.3,
        "ci95": [
         3.5,
         5.3
        ]
       },
       "5,001-10,000": {
        "n": 112,
        "of": 2778,
        "pct": 4.0,
        "ci95": [
         3.4,
         4.8
        ]
       }
      },
      "top1000_minus_5001_10000": {
       "diff_pp": 2.3,
       "ci95_pp": [
        0.3,
        4.9
       ]
      },
      "trend_3_tiers": {
       "z": -1.89,
       "p": 0.059
      },
      "trend_10_bands": {
       "z": -3.16,
       "p": 0.0016
      },
      "by_1000_band_pct": {
       "1-1000": 6.4,
       "1001-2000": 4.4,
       "2001-3000": 5.1,
       "3001-4000": 4.1,
       "4001-5000": 3.7,
       "5001-6000": 5.5,
       "6001-7000": 6.2,
       "7001-8000": 3.9,
       "8001-9000": 3.8,
       "9001-10000": 1.0
      }
     }
    }
   },
   "denominators": {
    "GPTBot": {
     "readable_files": {
      "n": 849,
      "of": 5572,
      "pct": 15.2,
      "ci95": [
       14.3,
       16.2
      ]
     },
     "readable_plus_missing_file": {
      "n": 849,
      "of": 7003,
      "pct": 12.1,
      "ci95": [
       11.4,
       12.9
      ]
     },
     "all_10000_lower_bound": {
      "n": 849,
      "of": 10000,
      "pct": 8.5,
      "ci95": [
       8.0,
       9.1
      ]
     },
     "live_homepage_readable_or_missing": {
      "n": 690,
      "of": 5372,
      "pct": 12.8,
      "ci95": [
       12.0,
       13.8
      ]
     }
    },
    "OAI-SearchBot": {
     "readable_files": {
      "n": 415,
      "of": 5572,
      "pct": 7.4,
      "ci95": [
       6.8,
       8.2
      ]
     },
     "readable_plus_missing_file": {
      "n": 415,
      "of": 7003,
      "pct": 5.9,
      "ci95": [
       5.4,
       6.5
      ]
     },
     "all_10000_lower_bound": {
      "n": 415,
      "of": 10000,
      "pct": 4.2,
      "ci95": [
       3.8,
       4.6
      ]
     },
     "live_homepage_readable_or_missing": {
      "n": 337,
      "of": 5372,
      "pct": 6.3,
      "ci95": [
       5.7,
       7.0
      ]
     }
    },
    "ClaudeBot": {
     "readable_files": {
      "n": 788,
      "of": 5572,
      "pct": 14.1,
      "ci95": [
       13.3,
       15.1
      ]
     },
     "readable_plus_missing_file": {
      "n": 788,
      "of": 7003,
      "pct": 11.3,
      "ci95": [
       10.5,
       12.0
      ]
     },
     "all_10000_lower_bound": {
      "n": 788,
      "of": 10000,
      "pct": 7.9,
      "ci95": [
       7.4,
       8.4
      ]
     },
     "live_homepage_readable_or_missing": {
      "n": 622,
      "of": 5372,
      "pct": 11.6,
      "ci95": [
       10.8,
       12.5
      ]
     }
    },
    "Claude-SearchBot": {
     "readable_files": {
      "n": 398,
      "of": 5572,
      "pct": 7.1,
      "ci95": [
       6.5,
       7.8
      ]
     },
     "readable_plus_missing_file": {
      "n": 398,
      "of": 7003,
      "pct": 5.7,
      "ci95": [
       5.2,
       6.3
      ]
     },
     "all_10000_lower_bound": {
      "n": 398,
      "of": 10000,
      "pct": 4.0,
      "ci95": [
       3.6,
       4.4
      ]
     },
     "live_homepage_readable_or_missing": {
      "n": 301,
      "of": 5372,
      "pct": 5.6,
      "ci95": [
       5.0,
       6.3
      ]
     }
    },
    "any_ai": {
     "readable_files": {
      "n": 1154,
      "of": 5572,
      "pct": 20.7,
      "ci95": [
       19.7,
       21.8
      ]
     },
     "readable_plus_missing_file": {
      "n": 1154,
      "of": 7003,
      "pct": 16.5,
      "ci95": [
       15.6,
       17.4
      ]
     },
     "all_10000_lower_bound": {
      "n": 1154,
      "of": 10000,
      "pct": 11.5,
      "ci95": [
       10.9,
       12.2
      ]
     },
     "live_homepage_readable_or_missing": {
      "n": 946,
      "of": 5372,
      "pct": 17.6,
      "ci95": [
       16.6,
       18.7
      ]
     }
    }
   },
   "denominator_counts": {
    "attempted": 10000,
    "readable": 5572,
    "missing_4xx": 1431,
    "live_homepage": 5902,
    "live_homepage_readable": 4959,
    "live_homepage_readable_or_missing": 5372
   },
   "block_source": {
    "GPTBot": {
     "purpose": "training",
     "blocked": 849,
     "by_name": 676,
     "via_wildcard": 173,
     "via_wildcard_pct_of_blocks": 20.4
    },
    "OAI-SearchBot": {
     "purpose": "search",
     "blocked": 415,
     "by_name": 242,
     "via_wildcard": 173,
     "via_wildcard_pct_of_blocks": 41.7
    },
    "ChatGPT-User": {
     "purpose": "user",
     "blocked": 539,
     "by_name": 365,
     "via_wildcard": 174,
     "via_wildcard_pct_of_blocks": 32.3
    },
    "ClaudeBot": {
     "purpose": "training",
     "blocked": 788,
     "by_name": 601,
     "via_wildcard": 187,
     "via_wildcard_pct_of_blocks": 23.7
    },
    "Claude-SearchBot": {
     "purpose": "search",
     "blocked": 398,
     "by_name": 208,
     "via_wildcard": 190,
     "via_wildcard_pct_of_blocks": 47.7
    },
    "Claude-User": {
     "purpose": "user",
     "blocked": 410,
     "by_name": 219,
     "via_wildcard": 191,
     "via_wildcard_pct_of_blocks": 46.6
    },
    "anthropic-ai": {
     "purpose": "legacy",
     "blocked": 660,
     "by_name": 461,
     "via_wildcard": 199,
     "via_wildcard_pct_of_blocks": 30.2
    },
    "Claude-Web": {
     "purpose": "legacy",
     "blocked": 579,
     "by_name": 379,
     "via_wildcard": 200,
     "via_wildcard_pct_of_blocks": 34.5
    },
    "PerplexityBot": {
     "purpose": "search",
     "blocked": 607,
     "by_name": 429,
     "via_wildcard": 178,
     "via_wildcard_pct_of_blocks": 29.3
    },
    "Perplexity-User": {
     "purpose": "user",
     "blocked": 427,
     "by_name": 232,
     "via_wildcard": 195,
     "via_wildcard_pct_of_blocks": 45.7
    },
    "Google-Extended": {
     "purpose": "training",
     "blocked": 686,
     "by_name": 506,
     "via_wildcard": 180,
     "via_wildcard_pct_of_blocks": 26.2
    },
    "Applebot-Extended": {
     "purpose": "training",
     "blocked": 649,
     "by_name": 457,
     "via_wildcard": 192,
     "via_wildcard_pct_of_blocks": 29.6
    },
    "CCBot": {
     "purpose": "training",
     "blocked": 904,
     "by_name": 710,
     "via_wildcard": 194,
     "via_wildcard_pct_of_blocks": 21.5
    },
    "Bytespider": {
     "purpose": "training",
     "blocked": 855,
     "by_name": 657,
     "via_wildcard": 198,
     "via_wildcard_pct_of_blocks": 23.2
    },
    "Meta-ExternalAgent": {
     "purpose": "training",
     "blocked": 688,
     "by_name": 494,
     "via_wildcard": 194,
     "via_wildcard_pct_of_blocks": 28.2
    },
    "meta-externalfetcher": {
     "purpose": "user",
     "blocked": 463,
     "by_name": 266,
     "via_wildcard": 197,
     "via_wildcard_pct_of_blocks": 42.5
    },
    "Amazonbot": {
     "purpose": "training",
     "blocked": 649,
     "by_name": 458,
     "via_wildcard": 191,
     "via_wildcard_pct_of_blocks": 29.4
    },
    "cohere-ai": {
     "purpose": "legacy",
     "blocked": 653,
     "by_name": 453,
     "via_wildcard": 200,
     "via_wildcard_pct_of_blocks": 30.6
    },
    "DuckAssistBot": {
     "purpose": "search",
     "blocked": 433,
     "by_name": 236,
     "via_wildcard": 197,
     "via_wildcard_pct_of_blocks": 45.5
    },
    "MistralAI-User": {
     "purpose": "user",
     "blocked": 397,
     "by_name": 192,
     "via_wildcard": 205,
     "via_wildcard_pct_of_blocks": 51.6
    },
    "Googlebot": {
     "purpose": "control",
     "blocked": 128,
     "by_name": 8,
     "via_wildcard": 120,
     "via_wildcard_pct_of_blocks": 93.8
    },
    "Bingbot": {
     "purpose": "control",
     "blocked": 173,
     "by_name": 35,
     "via_wildcard": 138,
     "via_wildcard_pct_of_blocks": 79.8
    },
    "Applebot": {
     "purpose": "control",
     "blocked": 245,
     "by_name": 76,
     "via_wildcard": 169,
     "via_wildcard_pct_of_blocks": 69.0
    }
   },
   "block_source_by_purpose": {
    "training": {
     "blocks": 6068,
     "via_wildcard": 1509,
     "via_wildcard_pct": 24.9
    },
    "search": {
     "blocks": 1853,
     "via_wildcard": 738,
     "via_wildcard_pct": 39.8
    },
    "user": {
     "blocks": 2236,
     "via_wildcard": 962,
     "via_wildcard_pct": 43.0
    }
   },
   "oai_search_blocked_googlebot_allowed_source": {
    "n": 291,
    "oai_search_named": 241,
    "via_wildcard": 50,
    "googlebot_named": 82,
    "via_wildcard_pct": 17.2
   },
   "wildcard_root_block_files": {
    "n": 214,
    "pct": 3.8,
    "ci95": [
     3.4,
     4.4
    ],
    "name_any_ai_token": 59,
    "let_oai_search_in_by_name": 32,
    "let_any_ai_search_in_by_name": 38,
    "googlebot_allowed": 90,
    "name_no_ai_token": 155,
    "name_no_ai_token_pct": 72.4
   },
   "training_search_2x2": {
    "openai": {
     "training": "GPTBot",
     "search": "OAI-SearchBot",
     "cells": {
      "allow_training__allow_search": {
       "n": 4718,
       "of": 5572,
       "pct": 84.7,
       "ci95": [
        83.7,
        85.6
       ]
      },
      "allow_training__block_search": {
       "n": 5,
       "of": 5572,
       "pct": 0.1,
       "ci95": [
        0.0,
        0.2
       ]
      },
      "block_training__allow_search": {
       "n": 439,
       "of": 5572,
       "pct": 7.9,
       "ci95": [
        7.2,
        8.6
       ]
      },
      "block_training__block_search": {
       "n": 410,
       "of": 5572,
       "pct": 7.4,
       "ci95": [
        6.7,
        8.1
       ]
      }
     },
     "block_training_allow_search": {
      "n": 439,
      "search_crawler_named": 76,
      "search_crawler_named_pct": 17.3,
      "search_crawler_not_mentioned": 363,
      "not_mentioned_pct": 82.7,
      "search_named_ci95": [
       14.1,
       21.1
      ]
     }
    },
    "anthropic": {
     "training": "ClaudeBot",
     "search": "Claude-SearchBot",
     "cells": {
      "allow_training__allow_search": {
       "n": 4783,
       "of": 5572,
       "pct": 85.8,
       "ci95": [
        84.9,
        86.7
       ]
      },
      "allow_training__block_search": {
       "n": 1,
       "of": 5572,
       "pct": 0.0,
       "ci95": [
        0.0,
        0.1
       ]
      },
      "block_training__allow_search": {
       "n": 391,
       "of": 5572,
       "pct": 7.0,
       "ci95": [
        6.4,
        7.7
       ]
      },
      "block_training__block_search": {
       "n": 397,
       "of": 5572,
       "pct": 7.1,
       "ci95": [
        6.5,
        7.8
       ]
      }
     },
     "block_training_allow_search": {
      "n": 391,
      "search_crawler_named": 48,
      "search_crawler_named_pct": 12.3,
      "search_crawler_not_mentioned": 343,
      "not_mentioned_pct": 87.7,
      "search_named_ci95": [
       9.4,
       15.9
      ]
     }
    }
   },
   "path_restrictions": {
    "GPTBot": {
     "path_rules_wildcard_group": {
      "n": 3404,
      "of": 5572,
      "pct": 61.1,
      "ci95": [
       59.8,
       62.4
      ]
     },
     "no_disallow_rule": {
      "n": 1125,
      "of": 5572,
      "pct": 20.2,
      "ci95": [
       19.2,
       21.3
      ]
     },
     "path_rules_named_group": {
      "n": 194,
      "of": 5572,
      "pct": 3.5,
      "ci95": [
       3.0,
       4.0
      ]
     },
     "full_block": {
      "n": 849,
      "of": 5572,
      "pct": 15.2,
      "ci95": [
       14.3,
       16.2
      ]
     }
    },
    "OAI-SearchBot": {
     "path_rules_wildcard_group": {
      "n": 3801,
      "of": 5572,
      "pct": 68.2,
      "ci95": [
       67.0,
       69.4
      ]
     },
     "no_disallow_rule": {
      "n": 1170,
      "of": 5572,
      "pct": 21.0,
      "ci95": [
       19.9,
       22.1
      ]
     },
     "full_block": {
      "n": 415,
      "of": 5572,
      "pct": 7.4,
      "ci95": [
       6.8,
       8.2
      ]
     },
     "path_rules_named_group": {
      "n": 186,
      "of": 5572,
      "pct": 3.3,
      "ci95": [
       2.9,
       3.8
      ]
     }
    },
    "ClaudeBot": {
     "path_rules_wildcard_group": {
      "n": 3525,
      "of": 5572,
      "pct": 63.3,
      "ci95": [
       62.0,
       64.5
      ]
     },
     "no_disallow_rule": {
      "n": 1116,
      "of": 5572,
      "pct": 20.0,
      "ci95": [
       19.0,
       21.1
      ]
     },
     "path_rules_named_group": {
      "n": 143,
      "of": 5572,
      "pct": 2.6,
      "ci95": [
       2.2,
       3.0
      ]
     },
     "full_block": {
      "n": 788,
      "of": 5572,
      "pct": 14.1,
      "ci95": [
       13.3,
       15.1
      ]
     }
    },
    "Claude-SearchBot": {
     "path_rules_wildcard_group": {
      "n": 3982,
      "of": 5572,
      "pct": 71.5,
      "ci95": [
       70.3,
       72.6
      ]
     },
     "no_disallow_rule": {
      "n": 1106,
      "of": 5572,
      "pct": 19.8,
      "ci95": [
       18.8,
       20.9
      ]
     },
     "full_block": {
      "n": 398,
      "of": 5572,
      "pct": 7.1,
      "ci95": [
       6.5,
       7.8
      ]
     },
     "path_rules_named_group": {
      "n": 86,
      "of": 5572,
      "pct": 1.5,
      "ci95": [
       1.3,
       1.9
      ]
     }
    },
    "PerplexityBot": {
     "path_rules_wildcard_group": {
      "n": 3649,
      "of": 5572,
      "pct": 65.5,
      "ci95": [
       64.2,
       66.7
      ]
     },
     "no_disallow_rule": {
      "n": 1158,
      "of": 5572,
      "pct": 20.8,
      "ci95": [
       19.7,
       21.9
      ]
     },
     "path_rules_named_group": {
      "n": 158,
      "of": 5572,
      "pct": 2.8,
      "ci95": [
       2.4,
       3.3
      ]
     },
     "full_block": {
      "n": 607,
      "of": 5572,
      "pct": 10.9,
      "ci95": [
       10.1,
       11.7
      ]
     }
    },
    "Googlebot": {
     "path_rules_wildcard_group": {
      "n": 3936,
      "of": 5572,
      "pct": 70.6,
      "ci95": [
       69.4,
       71.8
      ]
     },
     "no_disallow_rule": {
      "n": 1143,
      "of": 5572,
      "pct": 20.5,
      "ci95": [
       19.5,
       21.6
      ]
     },
     "path_rules_named_group": {
      "n": 365,
      "of": 5572,
      "pct": 6.6,
      "ci95": [
       5.9,
       7.2
      ]
     },
     "full_block": {
      "n": 128,
      "of": 5572,
      "pct": 2.3,
      "ci95": [
       1.9,
       2.7
      ]
     }
    }
   },
   "paths_closed_to_ai_open_to_googlebot": {
    "GPTBot": {
     "sites_root_open_to_both_with_listed_paths": 3620,
     "some_path_closed_to_crawler_open_to_googlebot": {
      "n": 216,
      "of": 3620,
      "pct": 6.0,
      "ci95": [
       5.2,
       6.8
      ]
     }
    },
    "OAI-SearchBot": {
     "sites_root_open_to_both_with_listed_paths": 4009,
     "some_path_closed_to_crawler_open_to_googlebot": {
      "n": 228,
      "of": 4009,
      "pct": 5.7,
      "ci95": [
       5.0,
       6.4
      ]
     }
    },
    "ClaudeBot": {
     "sites_root_open_to_both_with_listed_paths": 3672,
     "some_path_closed_to_crawler_open_to_googlebot": {
      "n": 184,
      "of": 3672,
      "pct": 5.0,
      "ci95": [
       4.4,
       5.8
      ]
     }
    },
    "Claude-SearchBot": {
     "sites_root_open_to_both_with_listed_paths": 4020,
     "some_path_closed_to_crawler_open_to_googlebot": {
      "n": 203,
      "of": 4020,
      "pct": 5.0,
      "ci95": [
       4.4,
       5.8
      ]
     }
    },
    "PerplexityBot": {
     "sites_root_open_to_both_with_listed_paths": 3827,
     "some_path_closed_to_crawler_open_to_googlebot": {
      "n": 191,
      "of": 3827,
      "pct": 5.0,
      "ci95": [
       4.3,
       5.7
      ]
     }
    }
   }
  },
  "s1_v12_linkage": {
   "collected": "2026-09-26",
   "engines": {
    "chatgpt": {
     "cited_urls": 384,
     "in_top10k": 156,
     "own_domain_excluded": 0,
     "readable_robots": 137,
     "domains": 70,
     "search": {
      "token": "OAI-SearchBot",
      "blocked": 7,
      "pct": 5.1,
      "ci95": [
       2.5,
       10.2
      ],
      "expected_pct": 8.6,
      "ratio_to_expected": 0.59,
      "p_binom": 0.17,
      "blocked_domains": [
       "amazon.com",
       "cnbc.com",
       "nbcnews.com"
      ],
      "url_level": {
       "same_host_urls": 123,
       "url_disallowed": 0,
       "pct": 0.0,
       "ci95": [
        0.0,
        3.0
       ]
      }
     },
     "training": {
      "token": "GPTBot",
      "blocked": 14,
      "pct": 10.2,
      "ci95": [
       6.2,
       16.4
      ],
      "expected_pct": 17.0,
      "ratio_to_expected": 0.6,
      "p_binom": 0.031,
      "blocked_domains": [
       "amazon.com",
       "apnews.com",
       "cnbc.com",
       "go.com",
       "healthline.com",
       "macworld.com",
       "nbcnews.com",
       "tripadvisor.com"
      ],
      "url_level": {
       "same_host_urls": 123,
       "url_disallowed": 6,
       "pct": 4.9,
       "ci95": [
        2.3,
        10.2
       ]
      }
     },
     "search_disallowed_urls": {
      "n": 0,
      "domains": 0,
      "top_domains": [],
      "also_disallowed_for_googlebot": 0,
      "also_disallowed_for_bingbot": 0,
      "allowed_for_googlebot": 0,
      "user_fetcher": "ChatGPT-User",
      "allowed_for_user_fetcher": 0,
      "search_token_named": 0
     },
     "train_only_blocked": {
      "n": 7,
      "pct": 5.1,
      "expected_pct": 8.5
     }
    },
    "claude": {
     "cited_urls": 260,
     "in_top10k": 58,
     "own_domain_excluded": 0,
     "readable_robots": 55,
     "domains": 21,
     "search": {
      "token": "Claude-SearchBot",
      "blocked": 7,
      "pct": 12.7,
      "ci95": [
       6.3,
       24.0
      ],
      "expected_pct": 9.9,
      "ratio_to_expected": 1.28,
      "p_binom": 0.49,
      "blocked_domains": [
       "yelp.com"
      ],
      "url_level": {
       "same_host_urls": 50,
       "url_disallowed": 7,
       "pct": 14.0,
       "ci95": [
        7.0,
        26.2
       ]
      }
     },
     "training": {
      "token": "ClaudeBot",
      "blocked": 29,
      "pct": 52.7,
      "ci95": [
       39.8,
       65.3
      ],
      "expected_pct": 17.8,
      "ratio_to_expected": 2.96,
      "p_binom": 4.8e-09,
      "blocked_domains": [
       "cnbc.com",
       "forbes.com",
       "healthline.com",
       "tripadvisor.com",
       "usnews.com",
       "yahoo.com",
       "yelp.com"
      ],
      "url_level": {
       "same_host_urls": 50,
       "url_disallowed": 20,
       "pct": 40.0,
       "ci95": [
        27.6,
        53.8
       ]
      }
     },
     "search_disallowed_urls": {
      "n": 7,
      "domains": 1,
      "top_domains": [
       [
        "yelp.com",
        7
       ]
      ],
      "also_disallowed_for_googlebot": 0,
      "also_disallowed_for_bingbot": 0,
      "allowed_for_googlebot": 7,
      "user_fetcher": "Claude-User",
      "allowed_for_user_fetcher": 0,
      "search_token_named": 7
     },
     "train_only_blocked": {
      "n": 22,
      "pct": 40.0,
      "expected_pct": 8.0
     }
    },
    "perplexity": {
     "cited_urls": 1588,
     "in_top10k": 664,
     "own_domain_excluded": 0,
     "readable_robots": 572,
     "domains": 141,
     "search": {
      "token": "PerplexityBot",
      "blocked": 267,
      "pct": 46.7,
      "ci95": [
       42.6,
       50.8
      ],
      "expected_pct": 13.9,
      "ratio_to_expected": 3.36,
      "p_binom": 2.8e-79,
      "blocked_domains": [
       "allrecipes.com",
       "cnbc.com",
       "cnet.com",
       "cnn.com",
       "cntraveler.com",
       "csmonitor.com",
       "eater.com",
       "everydayhealth.com",
       "foodandwine.com",
       "forbes.com",
       "glamour.com",
       "gq.com",
       "health.com",
       "ign.com",
       "investopedia.com",
       "lifehacker.com",
       "marketwatch.com",
       "mashable.com",
       "nbcnews.com",
       "newsbreak.com",
       "nymag.com",
       "nypost.com",
       "nytimes.com",
       "pcworld.com",
       "reddit.com",
       "refinery29.com",
       "theguardian.com",
       "thespruce.com",
       "thetimes.com",
       "theverge.com",
       "today.com",
       "travelandleisure.com",
       "usatoday.com",
       "usnews.com",
       "verywellmind.com",
       "wired.com",
       "wsj.com",
       "yahoo.com",
       "yelp.com",
       "zdnet.com"
      ],
      "url_level": {
       "same_host_urls": 517,
       "url_disallowed": 209,
       "pct": 40.4,
       "ci95": [
        36.3,
        44.7
       ]
      }
     },
     "search_disallowed_urls": {
      "n": 209,
      "domains": 37,
      "top_domains": [
       [
        "forbes.com",
        61
       ],
       [
        "nytimes.com",
        15
       ],
       [
        "reddit.com",
        13
       ],
       [
        "wsj.com",
        11
       ],
       [
        "marketwatch.com",
        10
       ],
       [
        "investopedia.com",
        10
       ],
       [
        "cnn.com",
        9
       ],
       [
        "travelandleisure.com",
        6
       ],
       [
        "everydayhealth.com",
        6
       ],
       [
        "yelp.com",
        6
       ],
       [
        "cnet.com",
        5
       ],
       [
        "thetimes.com",
        4
       ],
       [
        "usnews.com",
        4
       ],
       [
        "usatoday.com",
        4
       ],
       [
        "nypost.com",
        4
       ]
      ],
      "also_disallowed_for_googlebot": 15,
      "also_disallowed_for_bingbot": 15,
      "allowed_for_googlebot": 194,
      "user_fetcher": "Perplexity-User",
      "allowed_for_user_fetcher": 74,
      "search_token_named": 170
     }
    },
    "gemini": {
     "cited_urls": 259,
     "in_top10k": 112,
     "own_domain_excluded": 65,
     "readable_robots": 41,
     "domains": 23,
     "search": {
      "token": "Googlebot",
      "blocked": 2,
      "pct": 4.9,
      "ci95": [
       1.3,
       16.1
      ],
      "expected_pct": 2.3,
      "ratio_to_expected": 2.15,
      "p_binom": 0.24,
      "blocked_domains": [
       "reddit.com"
      ],
      "url_level": {
       "same_host_urls": 41,
       "url_disallowed": 2,
       "pct": 4.9,
       "ci95": [
        1.3,
        16.1
       ]
      }
     },
     "training": {
      "token": "Google-Extended",
      "blocked": 2,
      "pct": 4.9,
      "ci95": [
       1.3,
       16.1
      ],
      "expected_pct": 15.3,
      "ratio_to_expected": 0.32,
      "p_binom": 0.079,
      "blocked_domains": [
       "reddit.com"
      ],
      "url_level": {
       "same_host_urls": 41,
       "url_disallowed": 2,
       "pct": 4.9,
       "ci95": [
        1.3,
        16.1
       ]
      }
     },
     "search_disallowed_urls": {
      "n": 2,
      "domains": 1,
      "top_domains": [
       [
        "reddit.com",
        2
       ]
      ],
      "also_disallowed_for_googlebot": 2,
      "also_disallowed_for_bingbot": 2,
      "allowed_for_googlebot": 0,
      "user_fetcher": null,
      "allowed_for_user_fetcher": null,
      "search_token_named": 0
     }
    },
    "ai_overviews": {
     "cited_urls": 2827,
     "in_top10k": 1748,
     "own_domain_excluded": 533,
     "readable_robots": 1113,
     "domains": 160,
     "search": {
      "token": "Googlebot",
      "blocked": 64,
      "pct": 5.8,
      "ci95": [
       4.5,
       7.3
      ],
      "expected_pct": 2.2,
      "ratio_to_expected": 2.62,
      "p_binom": 9.9e-12,
      "blocked_domains": [
       "reddit.com"
      ],
      "url_level": {
       "same_host_urls": 979,
       "url_disallowed": 64,
       "pct": 6.5,
       "ci95": [
        5.2,
        8.3
       ]
      }
     },
     "training": {
      "token": "Google-Extended",
      "blocked": 198,
      "pct": 17.8,
      "ci95": [
       15.7,
       20.1
      ],
      "expected_pct": 16.8,
      "ratio_to_expected": 1.06,
      "p_binom": 0.38,
      "blocked_domains": [
       "allrecipes.com",
       "amazon.com",
       "cnbc.com",
       "cnn.com",
       "cntraveler.com",
       "consumerreports.org",
       "ebsco.com",
       "glassdoor.com",
       "groupon.com",
       "imdb.com",
       "instagram.com",
       "investopedia.com",
       "lifewire.com",
       "marketwatch.com",
       "nerdwallet.com",
       "nytimes.com",
       "pinterest.com",
       "reddit.com",
       "sciencedirect.com",
       "seriouseats.com",
       "snapchat.com",
       "thespruce.com",
       "theverge.com",
       "tiktok.com",
       "tripadvisor.com",
       "usnews.com",
       "verywellhealth.com",
       "wsj.com",
       "yahoo.com",
       "yelp.com"
      ],
      "url_level": {
       "same_host_urls": 979,
       "url_disallowed": 175,
       "pct": 17.9,
       "ci95": [
        15.6,
        20.4
       ]
      }
     },
     "search_disallowed_urls": {
      "n": 64,
      "domains": 1,
      "top_domains": [
       [
        "reddit.com",
        64
       ]
      ],
      "also_disallowed_for_googlebot": 64,
      "also_disallowed_for_bingbot": 64,
      "allowed_for_googlebot": 0,
      "user_fetcher": null,
      "allowed_for_user_fetcher": null,
      "search_token_named": 0
     }
    }
   }
  }
 }
}