{
 "generated": "2026-10-01",
 "source": "data/october_rerun_20261001_status.json",
 "method": "Re-crawl of the same Tranco top 5,000 domains plus a re-probe of the 468-host Cloudflare-managed-block panel and its 300-host control group, each with one fetch per host per crawler from one IP address. See /guides/cloudflare-managed-robots-txt-retired for the original method and its limits.",
 "crawler_access_recheck": {
  "with_robots_txt": {
   "value": 2729,
   "september": 2771
  },
  "oai_searchbot_blocked": {
   "value": 239,
   "september": 241
  },
  "perplexitybot_blocked": {
   "value": 354,
   "september": 359
  }
 },
 "cloudflare_edge_panel": {
  "gpt_claude_refused_perplexity_ok_of_chrome_ok_cloudflare": {
   "value": "322 of 411",
   "september": "336 of 422"
  },
  "control_group_of_chrome_ok": {
   "value": "17 of 269",
   "september": "17 of 273"
  }
 },
 "managed_block_agents_panel_refused_of_468": {
  "gptbot": {
   "value": 343,
   "september": 350
  },
  "claudebot": {
   "value": 340,
   "september": 348
  },
  "ccbot": {
   "value": 334,
   "september": 342
  },
  "amazonbot": {
   "value": 345,
   "september": 354
  },
  "bytespider": {
   "value": 354,
   "september": 362
  },
  "meta_externalagent": {
   "value": 42,
   "september": 43
  }
 },
 "managed_block_agents_control_refused_of_300": {
  "gptbot": {
   "value": 24,
   "september": 26
  },
  "claudebot": {
   "value": 38,
   "september": 36
  },
  "ccbot": {
   "value": 15,
   "september": 15
  },
  "amazonbot": {
   "value": 27,
   "september": 26
  },
  "bytespider": {
   "value": 74,
   "september": 75
  },
  "meta_externalagent": {
   "value": 18,
   "september": 15
  }
 },
 "robots_txt_path_panel": {
  "gptbot": {
   "home_refused": 347,
   "robots_served": 324,
   "robots_refused": 7,
   "robots_other": 16,
   "september_home_refused": 351,
   "september_robots_served": 330
  },
  "claudebot": {
   "home_refused": 344,
   "robots_served": 316,
   "robots_refused": 13,
   "robots_other": 15,
   "september_home_refused": 347,
   "september_robots_served": 320
  },
  "ccbot": {
   "home_refused": 338,
   "robots_served": 319,
   "robots_refused": 3,
   "robots_other": 16,
   "september_home_refused": 341,
   "september_robots_served": 324
  }
 },
 "top5000_edge_probe_ai_specific_refusals": {
  "total": {
   "value": 50,
   "september": 56
  },
  "per_crawler": {
   "value": {
    "perplexitybot": 41,
    "oai-searchbot": 19,
    "claude-searchbot": 10
   },
   "september": {
    "perplexitybot": 48,
    "oai-searchbot": 20,
    "claude-searchbot": 9
   }
  }
 },
 "content_signal_still_served_of_readable": {
  "value": "25 of 489",
  "september": "20 of 491"
 },
 "legacy_managed_block_marker_still_served": {
  "value": 9
 },
 "panel_fetch_status_changed_of_754": {
  "value": 25
 }
}