{
 "generated": "2026-09-08",
 "question": "of sites stating Content-Signal ai-train=no, how many also name and Disallow at least one of the 12 training-purpose crawlers this site tracks in robots.txt itself -- i.e. how many back the Content-Signal preference with an enforceable rule for a crawler that does not read Content-Signal at all (as of this check, that is every crawler except Cloudflare's own AI Crawl Control product).",
 "methodology": "single pass over the same Tranco top-N sample used elsewhere on this site: one robots.txt fetch per domain, Content-Signal parsing and training-crawler block matching run against the identical fetched text, so the two checks can never disagree about which robots.txt they saw. Training crawlers = the 12 agents this site's own census labels purpose:training in tools/aicrawl_census.py.",
 "sample_size": 5000,
 "ai_train_no_adopters": 125,
 "backed_by_at_least_one_training_disallow": 99,
 "backed_pct": 79.2,
 "unbacked": 26,
 "unbacked_pct": 20.8,
 "backed_count_distribution": {
  "0": 26,
  "1": 1,
  "2": 1,
  "5": 3,
  "7": 1,
  "8": 82,
  "9": 3,
  "10": 2,
  "11": 3,
  "12": 3
 },
 "examples_unbacked": [
  {
   "domain": "linktr.ee",
   "rank": 265,
   "raw": "ai-train=no, search=yes"
  },
  {
   "domain": "trendmicro.com",
   "rank": 533,
   "raw": "search=yes, ai-input=yes, ai-train=no"
  },
  {
   "domain": "visma.com",
   "rank": 785,
   "raw": "search=yes, ai-input=yes, ai-train=no"
  },
  {
   "domain": "life360.com",
   "rank": 897,
   "raw": "ai-train=no, ai-input=yes, search=yes"
  },
  {
   "domain": "segment.io",
   "rank": 975,
   "raw": "ai-train=no, search=yes, ai-input=no"
  },
  {
   "domain": "usercentrics.eu",
   "rank": 1121,
   "raw": "ai-train=no, search=yes, ai-input=yes"
  },
  {
   "domain": "wpastra.com",
   "rank": 1488,
   "raw": "ai-train=no, search=yes, ai-input=yes"
  },
  {
   "domain": "twilio.com",
   "rank": 1771,
   "raw": "ai-train=no, search=yes, ai-input=no"
  },
  {
   "domain": "moodle.org",
   "rank": 1849,
   "raw": "ai-train=no, search=yes, ai-input=no"
  },
  {
   "domain": "segment.com",
   "rank": 2091,
   "raw": "ai-train=no, search=yes, ai-input=no"
  },
  {
   "domain": "coinmarketcap.com",
   "rank": 2174,
   "raw": "search=yes, ai-train=no, ai-input=yes"
  },
  {
   "domain": "laravel.com",
   "rank": 2225,
   "raw": "ai-train=no, search=yes, ai-input=yes"
  },
  {
   "domain": "cookiebot.com",
   "rank": 2282,
   "raw": "ai-train=no, search=yes, ai-input=yes"
  },
  {
   "domain": "railway.app",
   "rank": 2683,
   "raw": "search=yes, ai-input=yes, ai-train=no"
  },
  {
   "domain": "padlet.com",
   "rank": 2750,
   "raw": "search=yes, ai-train=no, ai-input=yes"
  },
  {
   "domain": "vercel.com",
   "rank": 3184,
   "raw": "search=yes, ai-input=yes, ai-train=no"
  },
  {
   "domain": "wistia.com",
   "rank": 3469,
   "raw": "search=yes, ai-input=yes, ai-train=no"
  },
  {
   "domain": "coingecko.com",
   "rank": 4090,
   "raw": "ai-train=no, search=yes, ai-input=yes"
  },
  {
   "domain": "mailerlite.com",
   "rank": 4183,
   "raw": "search=yes, ai-input=yes, ai-train=no"
  },
  {
   "domain": "nationalarchives.gov.uk",
   "rank": 4287,
   "raw": "ai-train=no, search=yes, ai-input=no"
  },
  {
   "domain": "x.ai",
   "rank": 4378,
   "raw": "ai-train=no, search=yes, ai-input=no"
  },
  {
   "domain": "dish.com",
   "rank": 4450,
   "raw": "ai-train=no, search=yes, ai-input=yes"
  },
  {
   "domain": "tanium.com",
   "rank": 4489,
   "raw": "ai-train=no, search=yes, ai-input=yes"
  },
  {
   "domain": "sendgrid.com",
   "rank": 4490,
   "raw": "ai-train=no, search=yes, ai-input=no"
  },
  {
   "domain": "foreignpolicy.com",
   "rank": 4597,
   "raw": "search=yes,ai-train=no"
  },
  {
   "domain": "freshworks.com",
   "rank": 4796,
   "raw": "ai-train=no"
  }
 ],
 "examples_backed": [
  {
   "domain": "launchpad.net",
   "rank": 262,
   "raw": "ai-train=no, search=yes, ai-input=no",
   "training_agents_named_and_blocked": [
    "gptbot",
    "claudebot",
    "google-extended",
    "ccbot",
    "meta-externalagent",
    "bytespider",
    "applebot-extended",
    "amazonbot",
    "cohere-ai",
    "diffbot",
    "omgili",
    "timpibot"
   ]
  },
  {
   "domain": "weibo.com",
   "rank": 287,
   "raw": "search=yes, ai-input=yes, ai-train=no",
   "training_agents_named_and_blocked": [
    "meta-externalagent",
    "bytespider"
   ]
  },
  {
   "domain": "patreon.com",
   "rank": 541,
   "raw": "search=yes,ai-train=no,use=reference",
   "training_agents_named_and_blocked": [
    "gptbot",
    "claudebot",
    "google-extended",
    "ccbot",
    "meta-externalagent",
    "bytespider",
    "applebot-extended",
    "amazonbot",
    "timpibot"
   ]
  },
  {
   "domain": "cookielaw.org",
   "rank": 575,
   "raw": "search=yes,ai-train=no,use=reference",
   "training_agents_named_and_blocked": [
    "gptbot",
    "claudebot",
    "google-extended",
    "ccbot",
    "meta-externalagent",
    "bytespider",
    "applebot-extended",
    "amazonbot"
   ]
  },
  {
   "domain": "kick.com",
   "rank": 704,
   "raw": "search=yes,ai-train=no,use=reference",
   "training_agents_named_and_blocked": [
    "gptbot",
    "claudebot",
    "google-extended",
    "ccbot",
    "meta-externalagent",
    "bytespider",
    "applebot-extended",
    "amazonbot"
   ]
  },
  {
   "domain": "theatlantic.com",
   "rank": 1008,
   "raw": "search=yes, ai-input=no, ai-train=no",
   "training_agents_named_and_blocked": [
    "claudebot",
    "google-extended",
    "ccbot",
    "meta-externalagent",
    "bytespider",
    "applebot-extended",
    "amazonbot",
    "cohere-ai",
    "diffbot",
    "omgili",
    "timpibot"
   ]
  },
  {
   "domain": "pimpbunny.com",
   "rank": 1161,
   "raw": "search=yes,ai-train=no,use=reference",
   "training_agents_named_and_blocked": [
    "gptbot",
    "claudebot",
    "google-extended",
    "ccbot",
    "meta-externalagent",
    "bytespider",
    "applebot-extended",
    "amazonbot",
    "cohere-ai",
    "diffbot",
    "timpibot"
   ]
  },
  {
   "domain": "spankbang.com",
   "rank": 1181,
   "raw": "search=yes,ai-train=no,use=reference",
   "training_agents_named_and_blocked": [
    "gptbot",
    "claudebot",
    "google-extended",
    "ccbot",
    "meta-externalagent",
    "bytespider",
    "applebot-extended",
    "amazonbot"
   ]
  },
  {
   "domain": "hugedomains.com",
   "rank": 1300,
   "raw": "search=yes,ai-train=no,use=reference",
   "training_agents_named_and_blocked": [
    "gptbot",
    "claudebot",
    "google-extended",
    "ccbot",
    "meta-externalagent",
    "bytespider",
    "applebot-extended",
    "amazonbot"
   ]
  },
  {
   "domain": "comix.to",
   "rank": 1364,
   "raw": "search=yes,ai-train=no,use=reference",
   "training_agents_named_and_blocked": [
    "gptbot",
    "claudebot",
    "google-extended",
    "ccbot",
    "meta-externalagent",
    "bytespider",
    "applebot-extended",
    "amazonbot"
   ]
  },
  {
   "domain": "dagbladet.no",
   "rank": 1382,
   "raw": "search=yes, ai-train=no, ai-input=no",
   "training_agents_named_and_blocked": [
    "gptbot",
    "claudebot",
    "google-extended",
    "ccbot",
    "meta-externalagent",
    "bytespider",
    "applebot-extended",
    "amazonbot",
    "cohere-ai"
   ]
  },
  {
   "domain": "spankbang.party",
   "rank": 1416,
   "raw": "search=yes,ai-train=no,use=reference",
   "training_agents_named_and_blocked": [
    "gptbot",
    "claudebot",
    "google-extended",
    "ccbot",
    "meta-externalagent",
    "bytespider",
    "applebot-extended",
    "amazonbot"
   ]
  },
  {
   "domain": "omg10.com",
   "rank": 1424,
   "raw": "search=yes,ai-train=no,use=reference",
   "training_agents_named_and_blocked": [
    "gptbot",
    "claudebot",
    "google-extended",
    "ccbot",
    "meta-externalagent",
    "bytespider",
    "applebot-extended",
    "amazonbot"
   ]
  },
  {
   "domain": "cardmarket.com",
   "rank": 1428,
   "raw": "search=yes,ai-train=no,use=reference",
   "training_agents_named_and_blocked": [
    "gptbot",
    "claudebot",
    "google-extended",
    "ccbot",
    "meta-externalagent",
    "bytespider",
    "applebot-extended",
    "amazonbot"
   ]
  },
  {
   "domain": "pussyspace.com",
   "rank": 1441,
   "raw": "search=yes,ai-train=no,use=reference",
   "training_agents_named_and_blocked": [
    "gptbot",
    "claudebot",
    "google-extended",
    "ccbot",
    "meta-externalagent",
    "bytespider",
    "applebot-extended",
    "amazonbot"
   ]
  },
  {
   "domain": "crn77.com",
   "rank": 1449,
   "raw": "search=yes,ai-train=no,use=reference",
   "training_agents_named_and_blocked": [
    "gptbot",
    "claudebot",
    "google-extended",
    "ccbot",
    "meta-externalagent",
    "bytespider",
    "applebot-extended",
    "amazonbot"
   ]
  },
  {
   "domain": "bblaa.com",
   "rank": 1469,
   "raw": "search=yes,ai-train=no,use=reference",
   "training_agents_named_and_blocked": [
    "gptbot",
    "claudebot",
    "google-extended",
    "ccbot",
    "meta-externalagent",
    "bytespider",
    "applebot-extended",
    "amazonbot"
   ]
  },
  {
   "domain": "dogdrip.net",
   "rank": 1471,
   "raw": "search=yes,ai-train=no,use=reference",
   "training_agents_named_and_blocked": [
    "gptbot",
    "claudebot",
    "google-extended",
    "ccbot",
    "meta-externalagent",
    "bytespider",
    "applebot-extended",
    "amazonbot"
   ]
  },
  {
   "domain": "toyhou.se",
   "rank": 1473,
   "raw": "search=yes,ai-train=no,use=reference",
   "training_agents_named_and_blocked": [
    "gptbot",
    "claudebot",
    "google-extended",
    "ccbot",
    "meta-externalagent",
    "bytespider",
    "applebot-extended",
    "amazonbot"
   ]
  },
  {
   "domain": "xnxx.health",
   "rank": 1476,
   "raw": "search=yes,ai-train=no,use=reference",
   "training_agents_named_and_blocked": [
    "gptbot",
    "claudebot",
    "google-extended",
    "ccbot",
    "meta-externalagent",
    "bytespider",
    "applebot-extended",
    "amazonbot"
   ]
  },
  {
   "domain": "myinstants.com",
   "rank": 1491,
   "raw": "search=yes,ai-train=no,use=reference",
   "training_agents_named_and_blocked": [
    "gptbot",
    "claudebot",
    "google-extended",
    "ccbot",
    "meta-externalagent",
    "bytespider",
    "applebot-extended",
    "amazonbot"
   ]
  },
  {
   "domain": "ay267.com",
   "rank": 1495,
   "raw": "search=yes,ai-train=no,use=reference",
   "training_agents_named_and_blocked": [
    "gptbot",
    "claudebot",
    "google-extended",
    "ccbot",
    "meta-externalagent",
    "bytespider",
    "applebot-extended",
    "amazonbot"
   ]
  },
  {
   "domain": "255md.com",
   "rank": 1507,
   "raw": "search=yes,ai-train=no,use=reference",
   "training_agents_named_and_blocked": [
    "gptbot",
    "claudebot",
    "google-extended",
    "ccbot",
    "meta-externalagent",
    "bytespider",
    "applebot-extended",
    "amazonbot"
   ]
  },
  {
   "domain": "shahvani.com",
   "rank": 1513,
   "raw": "search=yes,ai-train=no,use=reference",
   "training_agents_named_and_blocked": [
    "gptbot",
    "claudebot",
    "google-extended",
    "ccbot",
    "meta-externalagent",
    "bytespider",
    "applebot-extended",
    "amazonbot"
   ]
  },
  {
   "domain": "wnacg.com",
   "rank": 1530,
   "raw": "search=yes,ai-train=no,use=reference",
   "training_agents_named_and_blocked": [
    "gptbot",
    "claudebot",
    "google-extended",
    "ccbot",
    "meta-externalagent",
    "bytespider",
    "applebot-extended",
    "amazonbot"
   ]
  },
  {
   "domain": "myreadingmanga.info",
   "rank": 1556,
   "raw": "search=yes,ai-train=no,use=reference",
   "training_agents_named_and_blocked": [
    "gptbot",
    "claudebot",
    "google-extended",
    "ccbot",
    "meta-externalagent",
    "bytespider",
    "applebot-extended",
    "amazonbot"
   ]
  },
  {
   "domain": "nya2.com",
   "rank": 1568,
   "raw": "search=yes,ai-train=no,use=reference",
   "training_agents_named_and_blocked": [
    "gptbot",
    "claudebot",
    "google-extended",
    "ccbot",
    "meta-externalagent",
    "bytespider",
    "applebot-extended",
    "amazonbot"
   ]
  },
  {
   "domain": "imgsrc.ru",
   "rank": 1573,
   "raw": "ai-train=no, search=yes, ai-input=no",
   "training_agents_named_and_blocked": [
    "gptbot"
   ]
  },
  {
   "domain": "artfight.net",
   "rank": 1633,
   "raw": "search=yes,ai-train=no,use=reference",
   "training_agents_named_and_blocked": [
    "gptbot",
    "claudebot",
    "google-extended",
    "ccbot",
    "meta-externalagent",
    "bytespider",
    "applebot-extended",
    "amazonbot",
    "cohere-ai",
    "diffbot",
    "omgili",
    "timpibot"
   ]
  },
  {
   "domain": "sportfilm800.com",
   "rank": 1647,
   "raw": "search=yes,ai-train=no,use=reference",
   "training_agents_named_and_blocked": [
    "gptbot",
    "claudebot",
    "google-extended",
    "ccbot",
    "meta-externalagent",
    "bytespider",
    "applebot-extended",
    "amazonbot"
   ]
  }
 ]
}