{
 "title": "AI crawler policy changes since the last census",
 "page": "https://ai-visibility.lastminutedealshq.com/changes",
 "author": "Reese Calder",
 "license": "CC BY 4.0. Free to reuse with a link to the page above.",
 "generated": "2026-09-18",
 "baseline_date": "2026-09-07",
 "window": "2026-09-07/2026-09-18",
 "method": "Re-crawl of a 1132-domain high-signal slice (current_blockers 709, asymmetry_296_subset 296, tranco_top500 500), diffed against the 2026-09-07 baseline. Only found-vs-found robots.txt comparisons count; an unknown fetch on either side is skipped. cdn_template_churn (shared hosting-template noise, not an independent decision) is excluded from this export -- see tools/reese_census_diff.py for the full method and https://ai-visibility.lastminutedealshq.com/data/census.json for the underlying census.",
 "counts": {
  "search_crawler_newly_blocked": 5,
  "search_crawler_newly_opened": 7,
  "training_crawler_policy_shift_applebot_extended": 33,
  "other_smaller_changes": 17
 },
 "events": {
  "search_crawler_newly_blocked": [
   {
    "domain": "pussyspace.com",
    "rank": 1441,
    "newly_blocked": [
     "perplexitybot"
    ]
   },
   {
    "domain": "hdtube.porn",
    "rank": 1459,
    "newly_blocked": [
     "claude-searchbot"
    ]
   },
   {
    "domain": "sportfilm800.com",
    "rank": 1647,
    "newly_blocked": [
     "claude-searchbot",
     "oai-searchbot",
     "perplexitybot"
    ]
   },
   {
    "domain": "lectormangass.net",
    "rank": 1725,
    "newly_blocked": [
     "perplexitybot"
    ]
   },
   {
    "domain": "bpexch.live",
    "rank": 1739,
    "newly_blocked": [
     "claude-searchbot",
     "oai-searchbot",
     "perplexitybot"
    ]
   }
  ],
  "search_crawler_newly_opened": [
   {
    "domain": "weather.com",
    "rank": 305,
    "newly_opened": [
     "perplexitybot"
    ]
   },
   {
    "domain": "sohu.com",
    "rank": 456,
    "newly_opened": [
     "claude-searchbot",
     "oai-searchbot",
     "perplexitybot"
    ]
   },
   {
    "domain": "gazzetta.it",
    "rank": 1230,
    "newly_opened": [
     "perplexitybot"
    ]
   },
   {
    "domain": "ladies.de",
    "rank": 1564,
    "newly_opened": [
     "claude-searchbot",
     "oai-searchbot",
     "perplexitybot"
    ]
   },
   {
    "domain": "glassdoor.com",
    "rank": 2529,
    "newly_opened": [
     "perplexitybot"
    ]
   },
   {
    "domain": "flipboard.com",
    "rank": 3222,
    "newly_opened": [
     "claude-searchbot",
     "perplexitybot"
    ]
   },
   {
    "domain": "kompas.com",
    "rank": 4057,
    "newly_opened": [
     "claude-searchbot"
    ]
   }
  ],
  "training_crawler_applebot_extended_added": [
   {
    "domain": "netflix.com",
    "rank": 41
   },
   {
    "domain": "pinterest.com",
    "rank": 57
   },
   {
    "domain": "flickr.com",
    "rank": 181
   },
   {
    "domain": "meraki.com",
    "rank": 254
   },
   {
    "domain": "imdb.com",
    "rank": 256
   },
   {
    "domain": "wsj.com",
    "rank": 361
   },
   {
    "domain": "netflix.net",
    "rank": 363
   },
   {
    "domain": "nflxvideo.net",
    "rank": 429
   },
   {
    "domain": "pinimg.com",
    "rank": 700
   },
   {
    "domain": "academia.edu",
    "rank": 814
   },
   {
    "domain": "theverge.com",
    "rank": 833
   },
   {
    "domain": "amap.com",
    "rank": 1270
   },
   {
    "domain": "ilmeteo.it",
    "rank": 1381
   },
   {
    "domain": "marketwatch.com",
    "rank": 1879
   },
   {
    "domain": "lnkd.in",
    "rank": 2052
   },
   {
    "domain": "thesun.co.uk",
    "rank": 2138
   },
   {
    "domain": "vox.com",
    "rank": 2289
   },
   {
    "domain": "redditmedia.com",
    "rank": 2452
   },
   {
    "domain": "news.com.au",
    "rank": 2539
   },
   {
    "domain": "instacart.com",
    "rank": 2774
   },
   {
    "domain": "thetimes.co.uk",
    "rank": 2925
   },
   {
    "domain": "theregister.com",
    "rank": 2955
   },
   {
    "domain": "sas.com",
    "rank": 3212
   },
   {
    "domain": "thetimes.com",
    "rank": 3285
   },
   {
    "domain": "nymag.com",
    "rank": 3349
   },
   {
    "domain": "realestate.com.au",
    "rank": 3729
   },
   {
    "domain": "toyota.com",
    "rank": 3819
   },
   {
    "domain": "wordreference.com",
    "rank": 3847
   },
   {
    "domain": "barrons.com",
    "rank": 4568
   },
   {
    "domain": "theregister.co.uk",
    "rank": 4611
   },
   {
    "domain": "boxofficemojo.com",
    "rank": 4700
   },
   {
    "domain": "turbify.com",
    "rank": 4713
   },
   {
    "domain": "ebsco.com",
    "rank": 3587,
    "note": "added 5 agents at once including applebot-extended, not a clean single-agent add"
   }
  ],
  "other_smaller_changes": [
   {
    "domain": "github.com",
    "rank": 29,
    "added": [
     "bytespider"
    ],
    "removed": []
   },
   {
    "domain": "t-online.de",
    "rank": 210,
    "added": [],
    "removed": [
     "google-extended"
    ]
   },
   {
    "domain": "hostinger.com",
    "rank": 325,
    "added": [
     "bytespider",
     "ccbot"
    ],
    "removed": []
   },
   {
    "domain": "welt.de",
    "rank": 792,
    "added": [
     "amazonbot"
    ],
    "removed": []
   },
   {
    "domain": "corriere.it",
    "rank": 969,
    "added": [
     "chatgpt-user"
    ],
    "removed": []
   },
   {
    "domain": "franceinfo.fr",
    "rank": 1095,
    "added": [
     "diffbot",
     "omgili",
     "timpibot"
    ],
    "removed": []
   },
   {
    "domain": "pimpbunny.com",
    "rank": 1161,
    "added": [],
    "removed": [
     "amazonbot"
    ]
   },
   {
    "domain": "syosetu.com",
    "rank": 1392,
    "added": [
     "meta-externalagent"
    ],
    "removed": []
   },
   {
    "domain": "wnacg.com",
    "rank": 1530,
    "added": [
     "googlebot"
    ],
    "removed": [
     "amazonbot",
     "applebot-extended",
     "bytespider",
     "ccbot",
     "claudebot",
     "google-extended",
     "gptbot",
     "meta-externalagent"
    ]
   },
   {
    "domain": "syosetu.org",
    "rank": 1571,
    "added": [
     "meta-externalagent"
    ],
    "removed": []
   },
   {
    "domain": "artfight.net",
    "rank": 1633,
    "added": [],
    "removed": [
     "amazonbot"
    ]
   },
   {
    "domain": "consumerreports.org",
    "rank": 3287,
    "added": [
     "meta-externalagent"
    ],
    "removed": []
   },
   {
    "domain": "abc.es",
    "rank": 3498,
    "added": [
     "claude-user",
     "cohere-ai",
     "perplexity-user"
    ],
    "removed": [
     "google-extended"
    ]
   },
   {
    "domain": "smugmug.com",
    "rank": 3582,
    "added": [
     "google-extended"
    ],
    "removed": []
   },
   {
    "domain": "ebsco.com",
    "rank": 3587,
    "added": [
     "amazonbot",
     "applebot-extended",
     "cohere-ai",
     "google-extended",
     "omgili"
    ],
    "removed": []
   },
   {
    "domain": "tvtropes.org",
    "rank": 4229,
    "added": [
     "omgili"
    ],
    "removed": []
   },
   {
    "domain": "kakaku.com",
    "rank": 4529,
    "added": [],
    "removed": [
     "ccbot"
    ]
   }
  ]
 }
}